2024-07-28T19:02:31.741 INFO:root:teuthology version: 0.0.1.dev373+ga8aed60 2024-07-28T19:02:31.767 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-07-28T19:02:31.871 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7815560' kernel: kdb: 1 sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 workunit: branch: reef sha1: a3bbd7289877bdcce87fd1f79da1a2d6578dde36 owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 2915 sha1: 1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_a3bbd7289877bdcce87fd1f79da1a2d6578dde36/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: a3bbd7289877bdcce87fd1f79da1a2d6578dde36 targets: smithi100.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBA6YGRYmBhU+I3UFgCAJdOpJSghHCTbdmJaAutRY8BB0df1C/BR/5qc4SAltVOCKFwCZupI2/UMxW5KaMonuKBE= smithi138.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK6pXsVxktBSwyopv7g3CWGPdRobPUArTg8Nut/KQAM+r9pvRLLlykrLowFvbshv5bskCdMWvKbXYDaa5hUThUs= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: a8aed60fc62d4ff39d2216b360f018eba7518cfe timestamp: 2024-07-24_01:16:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.4074525 2024-07-28T19:02:31.871 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_a3bbd7289877bdcce87fd1f79da1a2d6578dde36/qa; will attempt to use it 2024-07-28T19:02:31.872 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_a3bbd7289877bdcce87fd1f79da1a2d6578dde36/qa/tasks 2024-07-28T19:02:31.872 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-07-28T19:02:31.873 INFO:teuthology.task.internal:Checking packages... 2024-07-28T19:02:31.894 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64' 2024-07-28T19:02:31.895 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-28T19:02:31.895 INFO:teuthology.packaging:ref: None 2024-07-28T19:02:31.895 INFO:teuthology.packaging:tag: None 2024-07-28T19:02:31.895 INFO:teuthology.packaging:branch: reef 2024-07-28T19:02:31.895 INFO:teuthology.packaging:sha1: 1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:02:31.895 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-07-28T19:02:32.125 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1767-ga3bbd728-1jammy 2024-07-28T19:02:32.127 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-07-28T19:02:32.134 INFO:teuthology.task.internal:no buildpackages task found 2024-07-28T19:02:32.134 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-07-28T19:02:32.144 INFO:teuthology.task.internal:Saving configuration 2024-07-28T19:02:32.166 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-07-28T19:02:32.173 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-07-28T19:02:32.193 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi100.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-07-28 18:56:56.854278', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP+e250dyIqwJDg+ANEwwzQqV5M6mOAixZXbdbNdUVX6'} 2024-07-28T19:02:32.220 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi138.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-07-28 18:56:56.852797', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBJdsA4mdi/9znRIhP2KuwelIB5nP+dGIGaIH4+hPYv'} 2024-07-28T19:02:32.221 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-07-28T19:02:32.227 INFO:teuthology.task.internal:roles: ubuntu@smithi100.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-07-28T19:02:32.227 INFO:teuthology.task.internal:roles: ubuntu@smithi138.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-07-28T19:02:32.227 INFO:teuthology.run_tasks:Running task console_log... 2024-07-28T19:02:32.290 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fa3cb73af80>, signals=[15]) 2024-07-28T19:02:32.291 INFO:teuthology.run_tasks:Running task internal.connect... 2024-07-28T19:02:32.297 INFO:teuthology.task.internal:Opening connections... 2024-07-28T19:02:32.297 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi100.front.sepia.ceph.com 2024-07-28T19:02:32.298 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:02:32.376 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi138.front.sepia.ceph.com 2024-07-28T19:02:32.377 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:02:32.456 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-07-28T19:02:32.462 DEBUG:teuthology.orchestra.run.smithi100:> uname -m 2024-07-28T19:02:32.468 INFO:teuthology.orchestra.run.smithi100.stdout:x86_64 2024-07-28T19:02:32.468 DEBUG:teuthology.orchestra.run.smithi100:> cat /etc/os-release 2024-07-28T19:02:32.519 INFO:teuthology.orchestra.run.smithi100.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-07-28T19:02:32.519 INFO:teuthology.orchestra.run.smithi100.stdout:NAME="Ubuntu" 2024-07-28T19:02:32.519 INFO:teuthology.orchestra.run.smithi100.stdout:VERSION_ID="22.04" 2024-07-28T19:02:32.519 INFO:teuthology.orchestra.run.smithi100.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-07-28T19:02:32.520 INFO:teuthology.orchestra.run.smithi100.stdout:VERSION_CODENAME=jammy 2024-07-28T19:02:32.520 INFO:teuthology.orchestra.run.smithi100.stdout:ID=ubuntu 2024-07-28T19:02:32.520 INFO:teuthology.orchestra.run.smithi100.stdout:ID_LIKE=debian 2024-07-28T19:02:32.520 INFO:teuthology.orchestra.run.smithi100.stdout:HOME_URL="https://www.ubuntu.com/" 2024-07-28T19:02:32.520 INFO:teuthology.orchestra.run.smithi100.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-07-28T19:02:32.520 INFO:teuthology.orchestra.run.smithi100.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-07-28T19:02:32.520 INFO:teuthology.orchestra.run.smithi100.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-07-28T19:02:32.520 INFO:teuthology.orchestra.run.smithi100.stdout:UBUNTU_CODENAME=jammy 2024-07-28T19:02:32.521 INFO:teuthology.lock.ops:Updating smithi100.front.sepia.ceph.com on lock server 2024-07-28T19:02:32.547 DEBUG:teuthology.orchestra.run.smithi138:> uname -m 2024-07-28T19:02:32.552 INFO:teuthology.orchestra.run.smithi138.stdout:x86_64 2024-07-28T19:02:32.552 DEBUG:teuthology.orchestra.run.smithi138:> cat /etc/os-release 2024-07-28T19:02:32.599 INFO:teuthology.orchestra.run.smithi138.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-07-28T19:02:32.600 INFO:teuthology.orchestra.run.smithi138.stdout:NAME="Ubuntu" 2024-07-28T19:02:32.600 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION_ID="22.04" 2024-07-28T19:02:32.600 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-07-28T19:02:32.600 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION_CODENAME=jammy 2024-07-28T19:02:32.600 INFO:teuthology.orchestra.run.smithi138.stdout:ID=ubuntu 2024-07-28T19:02:32.600 INFO:teuthology.orchestra.run.smithi138.stdout:ID_LIKE=debian 2024-07-28T19:02:32.600 INFO:teuthology.orchestra.run.smithi138.stdout:HOME_URL="https://www.ubuntu.com/" 2024-07-28T19:02:32.601 INFO:teuthology.orchestra.run.smithi138.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-07-28T19:02:32.601 INFO:teuthology.orchestra.run.smithi138.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-07-28T19:02:32.601 INFO:teuthology.orchestra.run.smithi138.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-07-28T19:02:32.601 INFO:teuthology.orchestra.run.smithi138.stdout:UBUNTU_CODENAME=jammy 2024-07-28T19:02:32.601 INFO:teuthology.lock.ops:Updating smithi138.front.sepia.ceph.com on lock server 2024-07-28T19:02:32.621 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-07-28T19:02:32.630 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-07-28T19:02:32.636 INFO:teuthology.task.internal:Checking for old test directory... 2024-07-28T19:02:32.636 DEBUG:teuthology.orchestra.run.smithi100:> test '!' -e /home/ubuntu/cephtest 2024-07-28T19:02:32.639 DEBUG:teuthology.orchestra.run.smithi138:> test '!' -e /home/ubuntu/cephtest 2024-07-28T19:02:32.647 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-07-28T19:02:32.653 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-07-28T19:02:32.653 DEBUG:teuthology.orchestra.run.smithi100:> test -z $(ls -A /var/lib/ceph) 2024-07-28T19:02:32.685 DEBUG:teuthology.orchestra.run.smithi138:> test -z $(ls -A /var/lib/ceph) 2024-07-28T19:02:32.704 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-07-28T19:02:32.758 INFO:teuthology.run_tasks:Running task kernel... 2024-07-28T19:02:32.774 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-07-28T19:02:32.775 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-07-28T19:02:32.775 DEBUG:teuthology.orchestra.run.smithi100:> test -f /run/.containerenv -o -f /.dockerenv 2024-07-28T19:02:32.776 DEBUG:teuthology.orchestra.run.smithi138:> test -f /run/.containerenv -o -f /.dockerenv 2024-07-28T19:02:32.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-28T19:02:32.780 DEBUG:teuthology.orchestra.run.smithi100:> uname -r 2024-07-28T19:02:32.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-28T19:02:32.781 DEBUG:teuthology.orchestra.run.smithi138:> uname -r 2024-07-28T19:02:32.828 INFO:teuthology.orchestra.run.smithi138.stdout:5.15.0-56-generic 2024-07-28T19:02:32.829 INFO:teuthology.orchestra.run.smithi100.stdout:5.15.0-56-generic 2024-07-28T19:02:32.829 INFO:teuthology.task.kernel:Running kernel on smithi138: 5.15.0-56-generic 2024-07-28T19:02:32.829 DEBUG:teuthology.orchestra.run.smithi138:> sudo apt-get clean 2024-07-28T19:02:32.829 INFO:teuthology.task.kernel:Running kernel on smithi100: 5.15.0-56-generic 2024-07-28T19:02:32.830 DEBUG:teuthology.orchestra.run.smithi100:> sudo apt-get clean 2024-07-28T19:02:32.958 DEBUG:teuthology.orchestra.run.smithi138:> sudo apt-get update 2024-07-28T19:02:32.985 DEBUG:teuthology.orchestra.run.smithi100:> sudo apt-get update 2024-07-28T19:02:33.086 INFO:teuthology.orchestra.run.smithi138.stdout:Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-07-28T19:02:33.202 INFO:teuthology.orchestra.run.smithi138.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-28T19:02:33.259 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-28T19:02:33.259 INFO:teuthology.orchestra.run.smithi100.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-07-28T19:02:33.284 INFO:teuthology.orchestra.run.smithi138.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-07-28T19:02:33.312 INFO:teuthology.orchestra.run.smithi138.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [513 kB] 2024-07-28T19:02:33.313 INFO:teuthology.orchestra.run.smithi100.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-07-28T19:02:33.382 INFO:teuthology.orchestra.run.smithi138.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,673 kB] 2024-07-28T19:02:33.419 INFO:teuthology.orchestra.run.smithi138.stdout:Get:6 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [279 kB] 2024-07-28T19:02:33.421 INFO:teuthology.orchestra.run.smithi138.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.0 kB] 2024-07-28T19:02:33.421 INFO:teuthology.orchestra.run.smithi138.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,171 kB] 2024-07-28T19:02:33.440 INFO:teuthology.orchestra.run.smithi138.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.3 kB] 2024-07-28T19:02:33.440 INFO:teuthology.orchestra.run.smithi138.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [372 kB] 2024-07-28T19:02:33.443 INFO:teuthology.orchestra.run.smithi138.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [572 B] 2024-07-28T19:02:33.443 INFO:teuthology.orchestra.run.smithi138.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [886 kB] 2024-07-28T19:02:33.449 INFO:teuthology.orchestra.run.smithi138.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [619 kB] 2024-07-28T19:02:33.453 INFO:teuthology.orchestra.run.smithi138.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [173 kB] 2024-07-28T19:02:33.454 INFO:teuthology.orchestra.run.smithi138.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [18.9 kB] 2024-07-28T19:02:33.455 INFO:teuthology.orchestra.run.smithi138.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-07-28T19:02:33.455 INFO:teuthology.orchestra.run.smithi138.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-07-28T19:02:33.484 INFO:teuthology.orchestra.run.smithi138.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-07-28T19:02:33.485 INFO:teuthology.orchestra.run.smithi138.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-07-28T19:02:33.639 INFO:teuthology.orchestra.run.smithi138.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-07-28T19:02:33.671 INFO:teuthology.orchestra.run.smithi100.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-07-28T19:02:33.692 INFO:teuthology.orchestra.run.smithi100.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [513 kB] 2024-07-28T19:02:33.778 INFO:teuthology.orchestra.run.smithi138.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [672 kB] 2024-07-28T19:02:33.809 INFO:teuthology.orchestra.run.smithi100.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [672 kB] 2024-07-28T19:02:33.970 INFO:teuthology.orchestra.run.smithi138.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,888 kB] 2024-07-28T19:02:33.982 INFO:teuthology.orchestra.run.smithi100.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,673 kB] 2024-07-28T19:02:34.001 INFO:teuthology.orchestra.run.smithi100.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,888 kB] 2024-07-28T19:02:34.125 INFO:teuthology.orchestra.run.smithi138.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [337 kB] 2024-07-28T19:02:34.139 INFO:teuthology.orchestra.run.smithi138.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.7 kB] 2024-07-28T19:02:34.140 INFO:teuthology.orchestra.run.smithi138.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [38.9 kB] 2024-07-28T19:02:34.141 INFO:teuthology.orchestra.run.smithi138.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,238 kB] 2024-07-28T19:02:34.153 INFO:teuthology.orchestra.run.smithi100.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [279 kB] 2024-07-28T19:02:34.156 INFO:teuthology.orchestra.run.smithi100.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [337 kB] 2024-07-28T19:02:34.165 INFO:teuthology.orchestra.run.smithi100.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.0 kB] 2024-07-28T19:02:34.165 INFO:teuthology.orchestra.run.smithi100.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,171 kB] 2024-07-28T19:02:34.170 INFO:teuthology.orchestra.run.smithi100.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.7 kB] 2024-07-28T19:02:34.171 INFO:teuthology.orchestra.run.smithi100.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,238 kB] 2024-07-28T19:02:34.206 INFO:teuthology.orchestra.run.smithi138.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [384 kB] 2024-07-28T19:02:34.214 INFO:teuthology.orchestra.run.smithi138.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [604 B] 2024-07-28T19:02:34.215 INFO:teuthology.orchestra.run.smithi138.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,108 kB] 2024-07-28T19:02:34.237 INFO:teuthology.orchestra.run.smithi100.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [38.9 kB] 2024-07-28T19:02:34.238 INFO:teuthology.orchestra.run.smithi100.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [384 kB] 2024-07-28T19:02:34.253 INFO:teuthology.orchestra.run.smithi138.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [723 kB] 2024-07-28T19:02:34.266 INFO:teuthology.orchestra.run.smithi138.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [258 kB] 2024-07-28T19:02:34.267 INFO:teuthology.orchestra.run.smithi100.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [604 B] 2024-07-28T19:02:34.267 INFO:teuthology.orchestra.run.smithi100.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.3 kB] 2024-07-28T19:02:34.267 INFO:teuthology.orchestra.run.smithi100.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,108 kB] 2024-07-28T19:02:34.268 INFO:teuthology.orchestra.run.smithi100.stdout:Get:20 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [372 kB] 2024-07-28T19:02:34.272 INFO:teuthology.orchestra.run.smithi138.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [25.4 kB] 2024-07-28T19:02:34.275 INFO:teuthology.orchestra.run.smithi100.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [572 B] 2024-07-28T19:02:34.275 INFO:teuthology.orchestra.run.smithi100.stdout:Get:22 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [886 kB] 2024-07-28T19:02:34.288 INFO:teuthology.orchestra.run.smithi100.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [723 kB] 2024-07-28T19:02:34.290 INFO:teuthology.orchestra.run.smithi138.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,744 B] 2024-07-28T19:02:34.290 INFO:teuthology.orchestra.run.smithi100.stdout:Get:24 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [619 kB] 2024-07-28T19:02:34.291 INFO:teuthology.orchestra.run.smithi138.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-07-28T19:02:34.291 INFO:teuthology.orchestra.run.smithi138.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-07-28T19:02:34.292 INFO:teuthology.orchestra.run.smithi138.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-07-28T19:02:34.299 INFO:teuthology.orchestra.run.smithi100.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [258 kB] 2024-07-28T19:02:34.303 INFO:teuthology.orchestra.run.smithi100.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [25.4 kB] 2024-07-28T19:02:34.303 INFO:teuthology.orchestra.run.smithi100.stdout:Get:27 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [173 kB] 2024-07-28T19:02:34.305 INFO:teuthology.orchestra.run.smithi100.stdout:Get:28 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [18.9 kB] 2024-07-28T19:02:34.318 INFO:teuthology.orchestra.run.smithi100.stdout:Get:29 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-07-28T19:02:34.318 INFO:teuthology.orchestra.run.smithi100.stdout:Get:30 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-07-28T19:02:34.321 INFO:teuthology.orchestra.run.smithi100.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,744 B] 2024-07-28T19:02:34.321 INFO:teuthology.orchestra.run.smithi100.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-07-28T19:02:34.355 INFO:teuthology.orchestra.run.smithi138.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.1 kB] 2024-07-28T19:02:34.357 INFO:teuthology.orchestra.run.smithi138.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.3 kB] 2024-07-28T19:02:34.358 INFO:teuthology.orchestra.run.smithi138.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.0 kB] 2024-07-28T19:02:34.358 INFO:teuthology.orchestra.run.smithi138.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-07-28T19:02:34.358 INFO:teuthology.orchestra.run.smithi138.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.2 kB] 2024-07-28T19:02:34.372 INFO:teuthology.orchestra.run.smithi138.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-07-28T19:02:34.383 INFO:teuthology.orchestra.run.smithi100.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-07-28T19:02:34.384 INFO:teuthology.orchestra.run.smithi100.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-07-28T19:02:34.384 INFO:teuthology.orchestra.run.smithi100.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.1 kB] 2024-07-28T19:02:34.385 INFO:teuthology.orchestra.run.smithi100.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.3 kB] 2024-07-28T19:02:34.387 INFO:teuthology.orchestra.run.smithi100.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.0 kB] 2024-07-28T19:02:34.399 INFO:teuthology.orchestra.run.smithi100.stdout:Get:38 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-07-28T19:02:34.400 INFO:teuthology.orchestra.run.smithi100.stdout:Get:39 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-07-28T19:02:34.402 INFO:teuthology.orchestra.run.smithi100.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-07-28T19:02:34.402 INFO:teuthology.orchestra.run.smithi100.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-07-28T19:02:34.403 INFO:teuthology.orchestra.run.smithi100.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.2 kB] 2024-07-28T19:02:34.453 INFO:teuthology.orchestra.run.smithi138.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-07-28T19:02:34.454 INFO:teuthology.orchestra.run.smithi138.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-07-28T19:02:34.483 INFO:teuthology.orchestra.run.smithi100.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-07-28T19:02:34.484 INFO:teuthology.orchestra.run.smithi100.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-07-28T19:02:39.768 INFO:teuthology.orchestra.run.smithi138.stdout:Fetched 15.1 MB in 2s (7,640 kB/s) 2024-07-28T19:02:40.341 INFO:teuthology.orchestra.run.smithi100.stdout:Fetched 15.1 MB in 3s (5,767 kB/s) 2024-07-28T19:02:41.268 INFO:teuthology.orchestra.run.smithi138.stdout:Reading package lists... 2024-07-28T19:02:41.286 DEBUG:teuthology.orchestra.run.smithi138:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-07-28T19:02:41.338 INFO:teuthology.orchestra.run.smithi138.stdout:Reading package lists... 2024-07-28T19:02:41.523 INFO:teuthology.orchestra.run.smithi138.stdout:Building dependency tree... 2024-07-28T19:02:41.524 INFO:teuthology.orchestra.run.smithi138.stdout:Reading state information... 2024-07-28T19:02:41.677 INFO:teuthology.orchestra.run.smithi100.stdout:Reading package lists... 2024-07-28T19:02:41.686 INFO:teuthology.orchestra.run.smithi138.stdout:The following packages were automatically installed and are no longer required: 2024-07-28T19:02:41.687 INFO:teuthology.orchestra.run.smithi138.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-28T19:02:41.687 INFO:teuthology.orchestra.run.smithi138.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-28T19:02:41.687 INFO:teuthology.orchestra.run.smithi138.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-28T19:02:41.687 INFO:teuthology.orchestra.run.smithi138.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-07-28T19:02:41.687 INFO:teuthology.orchestra.run.smithi138.stdout: python2.7-minimal 2024-07-28T19:02:41.687 INFO:teuthology.orchestra.run.smithi138.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-28T19:02:41.689 INFO:teuthology.orchestra.run.smithi138.stdout:The following additional packages will be installed: 2024-07-28T19:02:41.689 INFO:teuthology.orchestra.run.smithi138.stdout: linux-generic linux-headers-5.15.0-117 linux-headers-5.15.0-117-generic 2024-07-28T19:02:41.689 INFO:teuthology.orchestra.run.smithi138.stdout: linux-headers-generic linux-image-5.15.0-117-generic 2024-07-28T19:02:41.689 INFO:teuthology.orchestra.run.smithi138.stdout: linux-modules-5.15.0-117-generic linux-modules-extra-5.15.0-117-generic 2024-07-28T19:02:41.691 INFO:teuthology.orchestra.run.smithi138.stdout:Suggested packages: 2024-07-28T19:02:41.691 INFO:teuthology.orchestra.run.smithi138.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-07-28T19:02:41.691 INFO:teuthology.orchestra.run.smithi138.stdout:Recommended packages: 2024-07-28T19:02:41.691 INFO:teuthology.orchestra.run.smithi138.stdout: thermald 2024-07-28T19:02:41.694 DEBUG:teuthology.orchestra.run.smithi100:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-07-28T19:02:41.741 INFO:teuthology.orchestra.run.smithi138.stdout:The following NEW packages will be installed: 2024-07-28T19:02:41.741 INFO:teuthology.orchestra.run.smithi138.stdout: linux-headers-5.15.0-117 linux-headers-5.15.0-117-generic 2024-07-28T19:02:41.742 INFO:teuthology.orchestra.run.smithi138.stdout: linux-image-5.15.0-117-generic linux-modules-5.15.0-117-generic 2024-07-28T19:02:41.742 INFO:teuthology.orchestra.run.smithi138.stdout: linux-modules-extra-5.15.0-117-generic 2024-07-28T19:02:41.743 INFO:teuthology.orchestra.run.smithi138.stdout:The following packages will be upgraded: 2024-07-28T19:02:41.744 INFO:teuthology.orchestra.run.smithi138.stdout: linux-generic linux-headers-generic linux-image-generic 2024-07-28T19:02:41.756 INFO:teuthology.orchestra.run.smithi100.stdout:Reading package lists... 2024-07-28T19:02:41.922 INFO:teuthology.orchestra.run.smithi138.stdout:3 upgraded, 5 newly installed, 0 to remove and 330 not upgraded. 2024-07-28T19:02:41.922 INFO:teuthology.orchestra.run.smithi138.stdout:Need to get 113 MB of archives. 2024-07-28T19:02:41.923 INFO:teuthology.orchestra.run.smithi138.stdout:After this operation, 583 MB of additional disk space will be used. 2024-07-28T19:02:41.923 INFO:teuthology.orchestra.run.smithi138.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-117-generic amd64 5.15.0-117.127 [22.6 MB] 2024-07-28T19:02:41.962 INFO:teuthology.orchestra.run.smithi100.stdout:Building dependency tree... 2024-07-28T19:02:41.963 INFO:teuthology.orchestra.run.smithi100.stdout:Reading state information... 2024-07-28T19:02:42.136 INFO:teuthology.orchestra.run.smithi100.stdout:The following packages were automatically installed and are no longer required: 2024-07-28T19:02:42.136 INFO:teuthology.orchestra.run.smithi100.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-28T19:02:42.136 INFO:teuthology.orchestra.run.smithi100.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-28T19:02:42.136 INFO:teuthology.orchestra.run.smithi100.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-28T19:02:42.137 INFO:teuthology.orchestra.run.smithi100.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-07-28T19:02:42.137 INFO:teuthology.orchestra.run.smithi100.stdout: python2.7-minimal 2024-07-28T19:02:42.137 INFO:teuthology.orchestra.run.smithi100.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-28T19:02:42.138 INFO:teuthology.orchestra.run.smithi100.stdout:The following additional packages will be installed: 2024-07-28T19:02:42.138 INFO:teuthology.orchestra.run.smithi100.stdout: linux-generic linux-headers-5.15.0-117 linux-headers-5.15.0-117-generic 2024-07-28T19:02:42.138 INFO:teuthology.orchestra.run.smithi100.stdout: linux-headers-generic linux-image-5.15.0-117-generic 2024-07-28T19:02:42.138 INFO:teuthology.orchestra.run.smithi100.stdout: linux-modules-5.15.0-117-generic linux-modules-extra-5.15.0-117-generic 2024-07-28T19:02:42.139 INFO:teuthology.orchestra.run.smithi100.stdout:Suggested packages: 2024-07-28T19:02:42.139 INFO:teuthology.orchestra.run.smithi100.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-07-28T19:02:42.140 INFO:teuthology.orchestra.run.smithi100.stdout:Recommended packages: 2024-07-28T19:02:42.140 INFO:teuthology.orchestra.run.smithi100.stdout: thermald 2024-07-28T19:02:42.200 INFO:teuthology.orchestra.run.smithi100.stdout:The following NEW packages will be installed: 2024-07-28T19:02:42.200 INFO:teuthology.orchestra.run.smithi100.stdout: linux-headers-5.15.0-117 linux-headers-5.15.0-117-generic 2024-07-28T19:02:42.201 INFO:teuthology.orchestra.run.smithi100.stdout: linux-image-5.15.0-117-generic linux-modules-5.15.0-117-generic 2024-07-28T19:02:42.201 INFO:teuthology.orchestra.run.smithi100.stdout: linux-modules-extra-5.15.0-117-generic 2024-07-28T19:02:42.202 INFO:teuthology.orchestra.run.smithi100.stdout:The following packages will be upgraded: 2024-07-28T19:02:42.202 INFO:teuthology.orchestra.run.smithi100.stdout: linux-generic linux-headers-generic linux-image-generic 2024-07-28T19:02:42.381 INFO:teuthology.orchestra.run.smithi100.stdout:3 upgraded, 5 newly installed, 0 to remove and 330 not upgraded. 2024-07-28T19:02:42.382 INFO:teuthology.orchestra.run.smithi100.stdout:Need to get 113 MB of archives. 2024-07-28T19:02:42.382 INFO:teuthology.orchestra.run.smithi100.stdout:After this operation, 583 MB of additional disk space will be used. 2024-07-28T19:02:42.382 INFO:teuthology.orchestra.run.smithi100.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-117-generic amd64 5.15.0-117.127 [22.6 MB] 2024-07-28T19:02:43.291 INFO:teuthology.orchestra.run.smithi138.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-117-generic amd64 5.15.0-117.127 [11.5 MB] 2024-07-28T19:02:43.637 INFO:teuthology.orchestra.run.smithi138.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-117-generic amd64 5.15.0-117.127 [64.0 MB] 2024-07-28T19:02:43.807 INFO:teuthology.orchestra.run.smithi100.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-117-generic amd64 5.15.0-117.127 [11.5 MB] 2024-07-28T19:02:44.213 INFO:teuthology.orchestra.run.smithi100.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-117-generic amd64 5.15.0-117.127 [64.0 MB] 2024-07-28T19:02:45.771 INFO:teuthology.orchestra.run.smithi138.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.117.117 [1,694 B] 2024-07-28T19:02:45.771 INFO:teuthology.orchestra.run.smithi138.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.117.117 [2,504 B] 2024-07-28T19:02:45.771 INFO:teuthology.orchestra.run.smithi138.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-117 all 5.15.0-117.127 [12.3 MB] 2024-07-28T19:02:46.311 INFO:teuthology.orchestra.run.smithi138.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-117-generic amd64 5.15.0-117.127 [2,880 kB] 2024-07-28T19:02:46.402 INFO:teuthology.orchestra.run.smithi100.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.117.117 [1,694 B] 2024-07-28T19:02:46.402 INFO:teuthology.orchestra.run.smithi100.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.117.117 [2,504 B] 2024-07-28T19:02:46.402 INFO:teuthology.orchestra.run.smithi100.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-117 all 5.15.0-117.127 [12.3 MB] 2024-07-28T19:02:46.413 INFO:teuthology.orchestra.run.smithi138.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.117.117 [2,358 B] 2024-07-28T19:02:46.801 INFO:teuthology.orchestra.run.smithi138.stdout:Fetched 113 MB in 5s (24.3 MB/s) 2024-07-28T19:02:46.986 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package linux-modules-5.15.0-117-generic. 2024-07-28T19:02:47.323 INFO:teuthology.orchestra.run.smithi100.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-117-generic amd64 5.15.0-117.127 [2,880 kB] 2024-07-28T19:02:47.430 INFO:teuthology.orchestra.run.smithi100.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.117.117 [2,358 B] 2024-07-28T19:02:47.837 INFO:teuthology.orchestra.run.smithi100.stdout:Fetched 113 MB in 5s (21.7 MB/s) 2024-07-28T19:02:48.015 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package linux-modules-5.15.0-117-generic. 2024-07-28T19:02:49.065 INFO:teuthology.orchestra.run.smithi138.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-07-28T19:02:49.069 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../0-linux-modules-5.15.0-117-generic_5.15.0-117.127_amd64.deb ... 2024-07-28T19:02:49.175 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking linux-modules-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:02:50.097 INFO:teuthology.orchestra.run.smithi100.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-07-28T19:02:50.101 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../0-linux-modules-5.15.0-117-generic_5.15.0-117.127_amd64.deb ... 2024-07-28T19:02:50.212 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking linux-modules-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:02:51.601 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package linux-image-5.15.0-117-generic. 2024-07-28T19:02:51.622 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../1-linux-image-5.15.0-117-generic_5.15.0-117.127_amd64.deb ... 2024-07-28T19:02:51.708 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking linux-image-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:02:52.146 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-117-generic. 2024-07-28T19:02:52.166 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-117-generic_5.15.0-117.127_amd64.deb ... 2024-07-28T19:02:52.204 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking linux-modules-extra-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:02:52.713 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package linux-image-5.15.0-117-generic. 2024-07-28T19:02:52.734 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../1-linux-image-5.15.0-117-generic_5.15.0-117.127_amd64.deb ... 2024-07-28T19:02:52.816 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking linux-image-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:02:53.291 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-117-generic. 2024-07-28T19:02:53.311 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-117-generic_5.15.0-117.127_amd64.deb ... 2024-07-28T19:02:53.341 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking linux-modules-extra-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:02:58.931 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../3-linux-generic_5.15.0.117.117_amd64.deb ... 2024-07-28T19:02:59.081 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking linux-generic (5.15.0.117.117) over (5.15.0.56.54) ... 2024-07-28T19:02:59.456 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.117.117_amd64.deb ... 2024-07-28T19:02:59.585 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking linux-image-generic (5.15.0.117.117) over (5.15.0.56.54) ... 2024-07-28T19:02:59.825 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../3-linux-generic_5.15.0.117.117_amd64.deb ... 2024-07-28T19:02:59.904 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package linux-headers-5.15.0-117. 2024-07-28T19:02:59.914 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../5-linux-headers-5.15.0-117_5.15.0-117.127_all.deb ... 2024-07-28T19:02:59.950 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking linux-generic (5.15.0.117.117) over (5.15.0.56.54) ... 2024-07-28T19:02:59.954 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking linux-headers-5.15.0-117 (5.15.0-117.127) ... 2024-07-28T19:03:00.282 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.117.117_amd64.deb ... 2024-07-28T19:03:00.403 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking linux-image-generic (5.15.0.117.117) over (5.15.0.56.54) ... 2024-07-28T19:03:00.732 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package linux-headers-5.15.0-117. 2024-07-28T19:03:00.754 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../5-linux-headers-5.15.0-117_5.15.0-117.127_all.deb ... 2024-07-28T19:03:00.798 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking linux-headers-5.15.0-117 (5.15.0-117.127) ... 2024-07-28T19:03:06.069 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package linux-headers-5.15.0-117-generic. 2024-07-28T19:03:06.105 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../6-linux-headers-5.15.0-117-generic_5.15.0-117.127_amd64.deb ... 2024-07-28T19:03:06.143 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking linux-headers-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:06.299 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package linux-headers-5.15.0-117-generic. 2024-07-28T19:03:06.336 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../6-linux-headers-5.15.0-117-generic_5.15.0-117.127_amd64.deb ... 2024-07-28T19:03:06.373 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking linux-headers-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:09.658 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.117.117_amd64.deb ... 2024-07-28T19:03:09.783 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking linux-headers-generic (5.15.0.117.117) over (5.15.0.56.54) ... 2024-07-28T19:03:10.133 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up linux-headers-5.15.0-117 (5.15.0-117.127) ... 2024-07-28T19:03:10.272 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up linux-headers-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:10.398 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up linux-headers-generic (5.15.0.117.117) ... 2024-07-28T19:03:10.527 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up linux-image-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:10.635 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.117.117_amd64.deb ... 2024-07-28T19:03:10.760 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking linux-headers-generic (5.15.0.117.117) over (5.15.0.56.54) ... 2024-07-28T19:03:11.110 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up linux-headers-5.15.0-117 (5.15.0-117.127) ... 2024-07-28T19:03:11.215 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up linux-headers-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:11.324 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up linux-headers-generic (5.15.0.117.117) ... 2024-07-28T19:03:11.478 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up linux-image-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:11.853 INFO:teuthology.orchestra.run.smithi138.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-07-28T19:03:11.854 INFO:teuthology.orchestra.run.smithi138.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-07-28T19:03:11.854 INFO:teuthology.orchestra.run.smithi138.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-117-generic 2024-07-28T19:03:11.854 INFO:teuthology.orchestra.run.smithi138.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-117-generic 2024-07-28T19:03:12.029 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up linux-modules-extra-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:12.802 INFO:teuthology.orchestra.run.smithi100.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-07-28T19:03:12.802 INFO:teuthology.orchestra.run.smithi100.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-07-28T19:03:12.802 INFO:teuthology.orchestra.run.smithi100.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-117-generic 2024-07-28T19:03:12.803 INFO:teuthology.orchestra.run.smithi100.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-117-generic 2024-07-28T19:03:12.963 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up linux-modules-extra-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:13.748 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up linux-image-generic (5.15.0.117.117) ... 2024-07-28T19:03:13.890 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up linux-generic (5.15.0.117.117) ... 2024-07-28T19:03:14.016 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up linux-modules-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:14.406 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up linux-image-generic (5.15.0.117.117) ... 2024-07-28T19:03:14.532 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up linux-generic (5.15.0.117.117) ... 2024-07-28T19:03:14.658 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up linux-modules-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:15.818 INFO:teuthology.orchestra.run.smithi138.stdout:Processing triggers for linux-image-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:15.884 INFO:teuthology.orchestra.run.smithi138.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-07-28T19:03:15.884 INFO:teuthology.orchestra.run.smithi138.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-117-generic 2024-07-28T19:03:16.168 INFO:teuthology.orchestra.run.smithi100.stdout:Processing triggers for linux-image-5.15.0-117-generic (5.15.0-117.127) ... 2024-07-28T19:03:16.235 INFO:teuthology.orchestra.run.smithi100.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-07-28T19:03:16.235 INFO:teuthology.orchestra.run.smithi100.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-117-generic 2024-07-28T19:03:30.084 INFO:teuthology.orchestra.run.smithi138.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-07-28T19:03:30.084 INFO:teuthology.orchestra.run.smithi138.stdout:Sourcing file `/etc/default/grub' 2024-07-28T19:03:30.097 INFO:teuthology.orchestra.run.smithi138.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-07-28T19:03:30.164 INFO:teuthology.orchestra.run.smithi138.stdout:Generating grub configuration file ... 2024-07-28T19:03:30.364 INFO:teuthology.orchestra.run.smithi100.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-07-28T19:03:30.364 INFO:teuthology.orchestra.run.smithi100.stdout:Sourcing file `/etc/default/grub' 2024-07-28T19:03:30.393 INFO:teuthology.orchestra.run.smithi100.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-07-28T19:03:30.417 INFO:teuthology.orchestra.run.smithi100.stdout:Generating grub configuration file ... 2024-07-28T19:03:30.470 INFO:teuthology.orchestra.run.smithi138.stdout:Found linux image: /boot/vmlinuz-5.15.0-117-generic 2024-07-28T19:03:30.486 INFO:teuthology.orchestra.run.smithi138.stdout:Found initrd image: /boot/initrd.img-5.15.0-117-generic 2024-07-28T19:03:30.694 INFO:teuthology.orchestra.run.smithi100.stdout:Found linux image: /boot/vmlinuz-5.15.0-117-generic 2024-07-28T19:03:30.710 INFO:teuthology.orchestra.run.smithi100.stdout:Found initrd image: /boot/initrd.img-5.15.0-117-generic 2024-07-28T19:03:30.903 INFO:teuthology.orchestra.run.smithi138.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-07-28T19:03:30.908 INFO:teuthology.orchestra.run.smithi138.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-07-28T19:03:30.972 INFO:teuthology.orchestra.run.smithi138.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-07-28T19:03:30.978 INFO:teuthology.orchestra.run.smithi138.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-07-28T19:03:31.053 INFO:teuthology.orchestra.run.smithi100.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-07-28T19:03:31.058 INFO:teuthology.orchestra.run.smithi100.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-07-28T19:03:31.113 INFO:teuthology.orchestra.run.smithi138.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-07-28T19:03:31.113 INFO:teuthology.orchestra.run.smithi138.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-07-28T19:03:31.113 INFO:teuthology.orchestra.run.smithi138.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-07-28T19:03:31.116 INFO:teuthology.orchestra.run.smithi100.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-07-28T19:03:31.122 INFO:teuthology.orchestra.run.smithi100.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-07-28T19:03:31.166 INFO:teuthology.orchestra.run.smithi138.stdout:done 2024-07-28T19:03:31.288 INFO:teuthology.orchestra.run.smithi100.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-07-28T19:03:31.288 INFO:teuthology.orchestra.run.smithi100.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-07-28T19:03:31.288 INFO:teuthology.orchestra.run.smithi100.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-07-28T19:03:31.326 INFO:teuthology.orchestra.run.smithi100.stdout:done 2024-07-28T19:03:31.660 DEBUG:teuthology.orchestra.run.smithi138:> dpkg -s linux-image-generic 2024-07-28T19:03:31.681 INFO:teuthology.orchestra.run.smithi138.stdout:Package: linux-image-generic 2024-07-28T19:03:31.681 INFO:teuthology.orchestra.run.smithi138.stdout:Status: install ok installed 2024-07-28T19:03:31.681 INFO:teuthology.orchestra.run.smithi138.stdout:Priority: optional 2024-07-28T19:03:31.681 INFO:teuthology.orchestra.run.smithi138.stdout:Section: kernel 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Installed-Size: 21 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Maintainer: Ubuntu Kernel Team 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Architecture: amd64 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Source: linux-meta 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Version: 5.15.0.117.117 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-117), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Depends: linux-image-5.15.0-117-generic, linux-modules-extra-5.15.0-117-generic, linux-firmware, intel-microcode, amd64-microcode 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Recommends: thermald 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout:Description: Generic Linux kernel image 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout: This package will always depend on the latest generic kernel image 2024-07-28T19:03:31.682 INFO:teuthology.orchestra.run.smithi138.stdout: available. 2024-07-28T19:03:31.683 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-117-generic 2024-07-28T19:03:31.683 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-07-28T19:03:31.683 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-07-28T19:03:31.683 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-07-28T19:03:31.683 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi138.front.sepia.ceph.com, path=None, version=distro) 2024-07-28T19:03:31.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo apt-get clean 2024-07-28T19:03:31.773 DEBUG:teuthology.orchestra.run.smithi100:> dpkg -s linux-image-generic 2024-07-28T19:03:31.790 INFO:teuthology.orchestra.run.smithi100.stdout:Package: linux-image-generic 2024-07-28T19:03:31.790 INFO:teuthology.orchestra.run.smithi100.stdout:Status: install ok installed 2024-07-28T19:03:31.790 INFO:teuthology.orchestra.run.smithi100.stdout:Priority: optional 2024-07-28T19:03:31.790 INFO:teuthology.orchestra.run.smithi100.stdout:Section: kernel 2024-07-28T19:03:31.790 INFO:teuthology.orchestra.run.smithi100.stdout:Installed-Size: 21 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout:Maintainer: Ubuntu Kernel Team 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout:Architecture: amd64 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout:Source: linux-meta 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout:Version: 5.15.0.117.117 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-117), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout:Depends: linux-image-5.15.0-117-generic, linux-modules-extra-5.15.0-117-generic, linux-firmware, intel-microcode, amd64-microcode 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout:Recommends: thermald 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout:Description: Generic Linux kernel image 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout: This package will always depend on the latest generic kernel image 2024-07-28T19:03:31.791 INFO:teuthology.orchestra.run.smithi100.stdout: available. 2024-07-28T19:03:31.791 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-117-generic 2024-07-28T19:03:31.791 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-07-28T19:03:31.792 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-07-28T19:03:31.792 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-07-28T19:03:31.792 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi100.front.sepia.ceph.com, path=None, version=distro) 2024-07-28T19:03:31.792 DEBUG:teuthology.orchestra.run.smithi100:> sudo apt-get clean 2024-07-28T19:03:31.916 DEBUG:teuthology.orchestra.run.smithi138:> sudo apt-get update 2024-07-28T19:03:31.950 DEBUG:teuthology.orchestra.run.smithi100:> sudo apt-get update 2024-07-28T19:03:32.150 INFO:teuthology.orchestra.run.smithi138.stdout:Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-07-28T19:03:32.158 INFO:teuthology.orchestra.run.smithi138.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-28T19:03:32.205 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-28T19:03:32.205 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-07-28T19:03:32.231 INFO:teuthology.orchestra.run.smithi138.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-07-28T19:03:32.277 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-07-28T19:03:32.313 INFO:teuthology.orchestra.run.smithi138.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-07-28T19:03:32.358 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-07-28T19:03:32.609 INFO:teuthology.orchestra.run.smithi138.stdout:Fetched 129 kB in 1s (242 kB/s) 2024-07-28T19:03:33.599 INFO:teuthology.orchestra.run.smithi138.stdout:Reading package lists... 2024-07-28T19:03:33.618 DEBUG:teuthology.orchestra.run.smithi138:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-07-28T19:03:33.636 INFO:teuthology.orchestra.run.smithi100.stdout:Reading package lists... 2024-07-28T19:03:33.653 DEBUG:teuthology.orchestra.run.smithi100:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-07-28T19:03:33.671 INFO:teuthology.orchestra.run.smithi138.stdout:Reading package lists... 2024-07-28T19:03:33.717 INFO:teuthology.orchestra.run.smithi100.stdout:Reading package lists... 2024-07-28T19:03:33.870 INFO:teuthology.orchestra.run.smithi138.stdout:Building dependency tree... 2024-07-28T19:03:33.871 INFO:teuthology.orchestra.run.smithi138.stdout:Reading state information... 2024-07-28T19:03:33.922 INFO:teuthology.orchestra.run.smithi100.stdout:Building dependency tree... 2024-07-28T19:03:33.922 INFO:teuthology.orchestra.run.smithi100.stdout:Reading state information... 2024-07-28T19:03:34.037 INFO:teuthology.orchestra.run.smithi138.stdout:linux-image-generic is already the newest version (5.15.0.117.117). 2024-07-28T19:03:34.037 INFO:teuthology.orchestra.run.smithi138.stdout:The following packages were automatically installed and are no longer required: 2024-07-28T19:03:34.037 INFO:teuthology.orchestra.run.smithi138.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-28T19:03:34.037 INFO:teuthology.orchestra.run.smithi138.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-28T19:03:34.037 INFO:teuthology.orchestra.run.smithi138.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-28T19:03:34.037 INFO:teuthology.orchestra.run.smithi138.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-07-28T19:03:34.038 INFO:teuthology.orchestra.run.smithi138.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-07-28T19:03:34.038 INFO:teuthology.orchestra.run.smithi138.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-28T19:03:34.077 INFO:teuthology.orchestra.run.smithi138.stdout:0 upgraded, 0 newly installed, 0 to remove and 330 not upgraded. 2024-07-28T19:03:34.078 DEBUG:teuthology.orchestra.run.smithi138:> dpkg -s linux-image-generic 2024-07-28T19:03:34.094 INFO:teuthology.orchestra.run.smithi100.stdout:linux-image-generic is already the newest version (5.15.0.117.117). 2024-07-28T19:03:34.094 INFO:teuthology.orchestra.run.smithi100.stdout:The following packages were automatically installed and are no longer required: 2024-07-28T19:03:34.094 INFO:teuthology.orchestra.run.smithi100.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-28T19:03:34.094 INFO:teuthology.orchestra.run.smithi100.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-28T19:03:34.094 INFO:teuthology.orchestra.run.smithi100.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-28T19:03:34.095 INFO:teuthology.orchestra.run.smithi100.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-07-28T19:03:34.095 INFO:teuthology.orchestra.run.smithi100.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-07-28T19:03:34.095 INFO:teuthology.orchestra.run.smithi100.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-28T19:03:34.134 INFO:teuthology.orchestra.run.smithi138.stdout:Package: linux-image-generic 2024-07-28T19:03:34.134 INFO:teuthology.orchestra.run.smithi138.stdout:Status: install ok installed 2024-07-28T19:03:34.134 INFO:teuthology.orchestra.run.smithi138.stdout:Priority: optional 2024-07-28T19:03:34.134 INFO:teuthology.orchestra.run.smithi138.stdout:Section: kernel 2024-07-28T19:03:34.134 INFO:teuthology.orchestra.run.smithi138.stdout:Installed-Size: 21 2024-07-28T19:03:34.134 INFO:teuthology.orchestra.run.smithi138.stdout:Maintainer: Ubuntu Kernel Team 2024-07-28T19:03:34.134 INFO:teuthology.orchestra.run.smithi138.stdout:Architecture: amd64 2024-07-28T19:03:34.134 INFO:teuthology.orchestra.run.smithi138.stdout:Source: linux-meta 2024-07-28T19:03:34.135 INFO:teuthology.orchestra.run.smithi138.stdout:Version: 5.15.0.117.117 2024-07-28T19:03:34.135 INFO:teuthology.orchestra.run.smithi138.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-117), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-07-28T19:03:34.135 INFO:teuthology.orchestra.run.smithi138.stdout:Depends: linux-image-5.15.0-117-generic, linux-modules-extra-5.15.0-117-generic, linux-firmware, intel-microcode, amd64-microcode 2024-07-28T19:03:34.135 INFO:teuthology.orchestra.run.smithi138.stdout:Recommends: thermald 2024-07-28T19:03:34.135 INFO:teuthology.orchestra.run.smithi138.stdout:Description: Generic Linux kernel image 2024-07-28T19:03:34.135 INFO:teuthology.orchestra.run.smithi138.stdout: This package will always depend on the latest generic kernel image 2024-07-28T19:03:34.135 INFO:teuthology.orchestra.run.smithi138.stdout: available. 2024-07-28T19:03:34.135 DEBUG:teuthology.orchestra.run.smithi138:> mktemp 2024-07-28T19:03:34.136 INFO:teuthology.orchestra.run.smithi100.stdout:0 upgraded, 0 newly installed, 0 to remove and 330 not upgraded. 2024-07-28T19:03:34.137 DEBUG:teuthology.orchestra.run.smithi100:> dpkg -s linux-image-generic 2024-07-28T19:03:34.152 INFO:teuthology.orchestra.run.smithi100.stdout:Package: linux-image-generic 2024-07-28T19:03:34.152 INFO:teuthology.orchestra.run.smithi100.stdout:Status: install ok installed 2024-07-28T19:03:34.152 INFO:teuthology.orchestra.run.smithi100.stdout:Priority: optional 2024-07-28T19:03:34.152 INFO:teuthology.orchestra.run.smithi100.stdout:Section: kernel 2024-07-28T19:03:34.152 INFO:teuthology.orchestra.run.smithi100.stdout:Installed-Size: 21 2024-07-28T19:03:34.152 INFO:teuthology.orchestra.run.smithi100.stdout:Maintainer: Ubuntu Kernel Team 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout:Architecture: amd64 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout:Source: linux-meta 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout:Version: 5.15.0.117.117 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-117), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout:Depends: linux-image-5.15.0-117-generic, linux-modules-extra-5.15.0-117-generic, linux-firmware, intel-microcode, amd64-microcode 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout:Recommends: thermald 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout:Description: Generic Linux kernel image 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout: This package will always depend on the latest generic kernel image 2024-07-28T19:03:34.153 INFO:teuthology.orchestra.run.smithi100.stdout: available. 2024-07-28T19:03:34.153 DEBUG:teuthology.orchestra.run.smithi100:> mktemp 2024-07-28T19:03:34.179 INFO:teuthology.orchestra.run.smithi138.stdout:/tmp/tmp.VkkBCcSE0s 2024-07-28T19:03:34.179 DEBUG:teuthology.orchestra.run.smithi138:> sudo cp /boot/grub/grub.cfg /tmp/tmp.VkkBCcSE0s 2024-07-28T19:03:34.198 INFO:teuthology.orchestra.run.smithi100.stdout:/tmp/tmp.90IwBjriNb 2024-07-28T19:03:34.198 DEBUG:teuthology.orchestra.run.smithi100:> sudo cp /boot/grub/grub.cfg /tmp/tmp.90IwBjriNb 2024-07-28T19:03:34.235 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 0666 /tmp/tmp.VkkBCcSE0s 2024-07-28T19:03:34.253 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod 0666 /tmp/tmp.90IwBjriNb 2024-07-28T19:03:34.359 DEBUG:teuthology.orchestra.remote:smithi138:/tmp/tmp.VkkBCcSE0s is 10KB 2024-07-28T19:03:34.371 DEBUG:teuthology.orchestra.run.smithi138:> rm -fr /tmp/tmp.VkkBCcSE0s 2024-07-28T19:03:34.375 DEBUG:teuthology.orchestra.remote:smithi100:/tmp/tmp.90IwBjriNb is 10KB 2024-07-28T19:03:34.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-07-28T19:03:34.388 DEBUG:teuthology.orchestra.run.smithi100:> rm -fr /tmp/tmp.90IwBjriNb 2024-07-28T19:03:34.393 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-07-28T19:03:34.431 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:03:34.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-07-28T19:03:34.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-07-28T19:03:34.446 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:03:34.446 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-07-28T19:03:34.446 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-07-28T19:03:34.507 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-117-generic 2024-07-28T19:03:34.508 DEBUG:teuthology.orchestra.run.smithi138:> sudo update-grub 2024-07-28T19:03:34.513 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-117-generic 2024-07-28T19:03:34.514 DEBUG:teuthology.orchestra.run.smithi100:> sudo update-grub 2024-07-28T19:03:35.753 INFO:teuthology.orchestra.run.smithi100.stderr:Sourcing file `/etc/default/grub' 2024-07-28T19:03:35.754 INFO:teuthology.orchestra.run.smithi100.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-07-28T19:03:35.756 INFO:teuthology.orchestra.run.smithi100.stderr:Generating grub configuration file ... 2024-07-28T19:03:35.890 INFO:teuthology.orchestra.run.smithi138.stderr:Sourcing file `/etc/default/grub' 2024-07-28T19:03:35.891 INFO:teuthology.orchestra.run.smithi138.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-07-28T19:03:35.893 INFO:teuthology.orchestra.run.smithi138.stderr:Generating grub configuration file ... 2024-07-28T19:03:35.964 INFO:teuthology.orchestra.run.smithi100.stderr:Found linux image: /boot/vmlinuz-5.15.0-117-generic 2024-07-28T19:03:35.971 INFO:teuthology.orchestra.run.smithi100.stderr:Found initrd image: /boot/initrd.img-5.15.0-117-generic 2024-07-28T19:03:36.052 INFO:teuthology.orchestra.run.smithi138.stderr:Found linux image: /boot/vmlinuz-5.15.0-117-generic 2024-07-28T19:03:36.057 INFO:teuthology.orchestra.run.smithi138.stderr:Found initrd image: /boot/initrd.img-5.15.0-117-generic 2024-07-28T19:03:36.195 INFO:teuthology.orchestra.run.smithi100.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-07-28T19:03:36.200 INFO:teuthology.orchestra.run.smithi100.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-07-28T19:03:36.241 INFO:teuthology.orchestra.run.smithi100.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-07-28T19:03:36.246 INFO:teuthology.orchestra.run.smithi100.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-07-28T19:03:36.363 INFO:teuthology.orchestra.run.smithi138.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-07-28T19:03:36.369 INFO:teuthology.orchestra.run.smithi138.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-07-28T19:03:36.373 INFO:teuthology.orchestra.run.smithi100.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-07-28T19:03:36.373 INFO:teuthology.orchestra.run.smithi100.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-07-28T19:03:36.373 INFO:teuthology.orchestra.run.smithi100.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-07-28T19:03:36.395 INFO:teuthology.orchestra.run.smithi100.stderr:done 2024-07-28T19:03:36.404 DEBUG:teuthology.orchestra.run.smithi100:> sudo shutdown -r now 2024-07-28T19:03:36.410 INFO:teuthology.orchestra.run.smithi138.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-07-28T19:03:36.415 INFO:teuthology.orchestra.run.smithi138.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-07-28T19:03:36.534 INFO:teuthology.orchestra.run.smithi138.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-07-28T19:03:36.534 INFO:teuthology.orchestra.run.smithi138.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-07-28T19:03:36.534 INFO:teuthology.orchestra.run.smithi138.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-07-28T19:03:36.548 INFO:teuthology.orchestra.run.smithi138.stderr:done 2024-07-28T19:03:36.557 DEBUG:teuthology.orchestra.run.smithi138:> sudo shutdown -r now 2024-07-28T19:04:06.435 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-07-28T19:04:06.435 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-07-28T19:04:06.436 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:04:06.559 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-07-28T19:04:06.560 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-07-28T19:04:06.560 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:04:40.927 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.100 2024-07-28T19:04:41.182 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.138 2024-07-28T19:04:49.934 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-07-28T19:04:49.936 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:04:50.184 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-07-28T19:04:50.184 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:05:08.319 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.100 2024-07-28T19:05:08.578 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.138 2024-07-28T19:05:20.330 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-07-28T19:05:20.331 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:05:20.580 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-07-28T19:05:20.580 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:05:20.670 DEBUG:teuthology.orchestra.run.smithi100:> true 2024-07-28T19:05:21.285 DEBUG:teuthology.orchestra.run.smithi138:> true 2024-07-28T19:05:21.329 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-07-28T19:05:21.329 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-117-generic"... 2024-07-28T19:05:21.329 DEBUG:teuthology.orchestra.run.smithi100:> uname -r 2024-07-28T19:05:21.378 INFO:teuthology.orchestra.run.smithi100.stdout:5.15.0-117-generic 2024-07-28T19:05:21.379 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-117-generic vs 5.15.0-117-generic 2024-07-28T19:05:21.379 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-07-28T19:05:21.379 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-07-28T19:05:22.218 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-07-28T19:05:22.218 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-117-generic"... 2024-07-28T19:05:22.218 DEBUG:teuthology.orchestra.run.smithi138:> uname -r 2024-07-28T19:05:22.223 INFO:teuthology.orchestra.run.smithi138.stdout:5.15.0-117-generic 2024-07-28T19:05:22.223 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-117-generic vs 5.15.0-117-generic 2024-07-28T19:05:22.223 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-07-28T19:05:22.224 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-07-28T19:05:22.380 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-07-28T19:05:22.380 DEBUG:teuthology.orchestra.run.smithi100:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-07-28T19:05:22.490 INFO:teuthology.orchestra.run.smithi100.stdout:ttyS1 2024-07-28T19:05:22.522 DEBUG:teuthology.parallel:result is None 2024-07-28T19:05:23.224 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-07-28T19:05:23.225 DEBUG:teuthology.orchestra.run.smithi138:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-07-28T19:05:23.479 INFO:teuthology.orchestra.run.smithi138.stdout:ttyS1 2024-07-28T19:05:23.505 DEBUG:teuthology.parallel:result is None 2024-07-28T19:05:23.505 INFO:teuthology.run_tasks:Running task internal.base... 2024-07-28T19:05:23.513 INFO:teuthology.task.internal:Creating test directory... 2024-07-28T19:05:23.513 DEBUG:teuthology.orchestra.run.smithi100:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-07-28T19:05:23.515 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-07-28T19:05:23.521 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-07-28T19:05:23.561 INFO:teuthology.run_tasks:Running task internal.archive... 2024-07-28T19:05:23.581 INFO:teuthology.task.internal:Creating archive directory... 2024-07-28T19:05:23.581 DEBUG:teuthology.orchestra.run.smithi100:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-07-28T19:05:23.584 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-07-28T19:05:23.701 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-07-28T19:05:23.709 INFO:teuthology.task.internal:Enabling coredump saving... 2024-07-28T19:05:23.709 DEBUG:teuthology.orchestra.run.smithi100:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-07-28T19:05:23.711 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-07-28T19:05:23.743 INFO:teuthology.orchestra.run.smithi100.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-07-28T19:05:23.751 INFO:teuthology.orchestra.run.smithi100.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-07-28T19:05:23.755 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-07-28T19:05:23.764 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-07-28T19:05:23.765 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-07-28T19:05:23.805 INFO:teuthology.task.internal:Configuring sudo... 2024-07-28T19:05:23.805 DEBUG:teuthology.orchestra.run.smithi100:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-07-28T19:05:23.808 DEBUG:teuthology.orchestra.run.smithi138:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-07-28T19:05:23.857 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-07-28T19:05:23.905 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-07-28T19:05:23.905 DEBUG:teuthology.orchestra.run.smithi100:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-07-28T19:05:23.908 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-07-28T19:05:23.913 DEBUG:teuthology.orchestra.run.smithi100:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-07-28T19:05:23.957 DEBUG:teuthology.orchestra.run.smithi100:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-07-28T19:05:24.005 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:05:24.005 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-07-28T19:05:24.074 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-07-28T19:05:24.080 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-07-28T19:05:24.127 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:05:24.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-07-28T19:05:24.296 DEBUG:teuthology.orchestra.run.smithi100:> sudo service rsyslog restart 2024-07-28T19:05:24.299 DEBUG:teuthology.orchestra.run.smithi138:> sudo service rsyslog restart 2024-07-28T19:05:24.408 INFO:teuthology.run_tasks:Running task internal.timer... 2024-07-28T19:05:24.416 INFO:teuthology.task.internal:Starting timer... 2024-07-28T19:05:24.416 INFO:teuthology.run_tasks:Running task pcp... 2024-07-28T19:05:24.511 INFO:teuthology.run_tasks:Running task selinux... 2024-07-28T19:05:24.631 DEBUG:teuthology.task.selinux:Excluding smithi100: OS 'ubuntu' does not support SELinux 2024-07-28T19:05:24.631 DEBUG:teuthology.task.selinux:Excluding smithi138: OS 'ubuntu' does not support SELinux 2024-07-28T19:05:24.631 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-07-28T19:05:24.631 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-07-28T19:05:24.631 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-07-28T19:05:24.631 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-07-28T19:05:24.689 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-07-28T19:05:24.700 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-07-28T19:05:24.701 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi100.front.sepia.ceph.com,smithi138.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-07-28T19:12:30.119 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi100.front.sepia.ceph.com'), Remote(name='ubuntu@smithi138.front.sepia.ceph.com')] 2024-07-28T19:12:30.120 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-07-28T19:12:30.122 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:12:30.202 DEBUG:teuthology.orchestra.run.smithi100:> true 2024-07-28T19:12:30.278 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-07-28T19:12:30.279 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-07-28T19:12:30.279 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-07-28T19:12:30.359 DEBUG:teuthology.orchestra.run.smithi138:> true 2024-07-28T19:12:30.435 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-07-28T19:12:30.435 INFO:teuthology.run_tasks:Running task clock... 2024-07-28T19:12:30.446 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-07-28T19:12:30.446 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-07-28T19:12:30.447 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-07-28T19:12:30.449 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-07-28T19:12:30.450 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Command line: ntpd -gq 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: ---------------------------------------------------- 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: ntp-4 is maintained by Network Time Foundation, 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: corporation. Support and training for ntp-4 are 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: available at https://www.nwtime.org/support 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: ---------------------------------------------------- 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: proto: precision = 0.104 usec (-23) 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: basedate set to 2022-02-04 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: gps base set to 2022-02-06 (week 2196) 2024-07-28T19:12:30.480 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-07-28T19:12:30.481 INFO:teuthology.orchestra.run.smithi100.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-07-28T19:12:30.481 INFO:teuthology.orchestra.run.smithi100.stderr:restrict ::: KOD does nothing without LIMITED. 2024-07-28T19:12:30.482 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: restrict ::: KOD does nothing without LIMITED. 2024-07-28T19:12:30.482 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Listen and drop on 0 v6wildcard [::]:123 2024-07-28T19:12:30.483 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-07-28T19:12:30.483 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Listen normally on 2 lo 127.0.0.1:123 2024-07-28T19:12:30.483 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Listen normally on 3 enp3s0f1 172.21.15.100:123 2024-07-28T19:12:30.483 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Listen normally on 4 lo [::1]:123 2024-07-28T19:12:30.483 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d733%5]:123 2024-07-28T19:12:30.483 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:30 ntpd[17697]: Listening on routing socket on fd #22 for interface updates 2024-07-28T19:12:30.504 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-07-28T19:12:30.505 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Command line: ntpd -gq 2024-07-28T19:12:30.505 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: ---------------------------------------------------- 2024-07-28T19:12:30.505 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: ntp-4 is maintained by Network Time Foundation, 2024-07-28T19:12:30.505 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-07-28T19:12:30.505 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: corporation. Support and training for ntp-4 are 2024-07-28T19:12:30.505 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: available at https://www.nwtime.org/support 2024-07-28T19:12:30.505 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: ---------------------------------------------------- 2024-07-28T19:12:30.506 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: proto: precision = 0.060 usec (-24) 2024-07-28T19:12:30.506 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: basedate set to 2022-02-04 2024-07-28T19:12:30.506 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: gps base set to 2022-02-06 (week 2196) 2024-07-28T19:12:30.506 INFO:teuthology.orchestra.run.smithi138.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-07-28T19:12:30.507 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-07-28T19:12:30.507 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: restrict ::: KOD does nothing without LIMITED. 2024-07-28T19:12:30.507 INFO:teuthology.orchestra.run.smithi138.stderr:restrict ::: KOD does nothing without LIMITED. 2024-07-28T19:12:30.508 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Listen and drop on 0 v6wildcard [::]:123 2024-07-28T19:12:30.508 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-07-28T19:12:30.508 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Listen normally on 2 lo 127.0.0.1:123 2024-07-28T19:12:30.508 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Listen normally on 3 enp3s0f1 172.21.15.138:123 2024-07-28T19:12:30.508 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Listen normally on 4 lo [::1]:123 2024-07-28T19:12:30.508 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:6f71%5]:123 2024-07-28T19:12:30.508 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:30 ntpd[17723]: Listening on routing socket on fd #22 for interface updates 2024-07-28T19:12:31.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:31.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:31.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:31 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:31.505 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:31 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:31.505 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:31 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:31.505 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:31 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:31.505 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:31 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:31.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:31 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:31.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:31 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:31.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:31 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:31.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:31 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:33.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:33.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:33.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:33.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:33.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:33.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:33.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:33.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:33.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:33.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:33.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:33.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:33 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:33.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:33.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:33 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:34.505 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:34 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:34.505 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:34 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:34.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:34 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:34.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:34 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:34.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:34 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:34.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:34 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:35.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:35.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:35 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:35.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:35.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:35.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:35.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:35.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:35.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:35 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:36.505 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:36 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:36.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:36 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:36.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:36 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:36.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:36 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:36.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:36 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:36.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:36 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:37.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.480 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:37.481 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:37.482 INFO:teuthology.orchestra.run.smithi100.stdout:28 Jul 19:12:37 ntpd[17697]: ntpd: time slew -0.001762 s 2024-07-28T19:12:37.482 INFO:teuthology.orchestra.run.smithi100.stdout:ntpd: time slew -0.001762s 2024-07-28T19:12:37.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-07-28T19:12:37.482 INFO:teuthology.orchestra.run.smithi100.stderr:28 Jul 19:12:37 ntpd[17697]: can't open /var/log/ntpstats/loopstats.20240728: Permission denied 2024-07-28T19:12:37.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.506 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:37.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:37.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-07-28T19:12:37.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: can't open /var/log/ntpstats/rawstats.20240728: Permission denied 2024-07-28T19:12:37.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-07-28T19:12:37.507 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: can't open /var/log/ntpstats/peerstats.20240728: Permission denied 2024-07-28T19:12:37.507 INFO:teuthology.orchestra.run.smithi138.stdout:28 Jul 19:12:37 ntpd[17723]: ntpd: time slew -0.001664 s 2024-07-28T19:12:37.507 INFO:teuthology.orchestra.run.smithi138.stdout:ntpd: time slew -0.001664s 2024-07-28T19:12:37.508 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-07-28T19:12:37.508 INFO:teuthology.orchestra.run.smithi138.stderr:28 Jul 19:12:37 ntpd[17723]: can't open /var/log/ntpstats/loopstats.20240728: Permission denied 2024-07-28T19:12:37.536 INFO:teuthology.orchestra.run.smithi100.stdout: remote refid st t when poll reach delay offset jitter 2024-07-28T19:12:37.537 INFO:teuthology.orchestra.run.smithi100.stdout:============================================================================== 2024-07-28T19:12:37.537 INFO:teuthology.orchestra.run.smithi100.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-28T19:12:37.537 INFO:teuthology.orchestra.run.smithi100.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-28T19:12:37.537 INFO:teuthology.orchestra.run.smithi100.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-28T19:12:37.537 INFO:teuthology.orchestra.run.smithi100.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-28T19:12:37.570 INFO:teuthology.orchestra.run.smithi138.stdout: remote refid st t when poll reach delay offset jitter 2024-07-28T19:12:37.570 INFO:teuthology.orchestra.run.smithi138.stdout:============================================================================== 2024-07-28T19:12:37.571 INFO:teuthology.orchestra.run.smithi138.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-28T19:12:37.571 INFO:teuthology.orchestra.run.smithi138.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-28T19:12:37.571 INFO:teuthology.orchestra.run.smithi138.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-28T19:12:37.571 INFO:teuthology.orchestra.run.smithi138.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-07-28T19:12:37.572 INFO:teuthology.run_tasks:Running task install... 2024-07-28T19:12:37.580 DEBUG:teuthology.task.install:project ceph 2024-07-28T19:12:37.581 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64'}} 2024-07-28T19:12:37.581 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64'} 2024-07-28T19:12:37.581 INFO:teuthology.task.install:Using flavor: default 2024-07-28T19:12:37.586 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-07-28T19:12:37.586 INFO:teuthology.task.install:extra packages: [] 2024-07-28T19:12:37.587 DEBUG:teuthology.orchestra.run.smithi100:> sudo apt-key list | grep Ceph 2024-07-28T19:12:37.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo apt-key list | grep Ceph 2024-07-28T19:12:37.645 INFO:teuthology.orchestra.run.smithi100.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-07-28T19:12:37.671 INFO:teuthology.orchestra.run.smithi138.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-07-28T19:12:37.673 INFO:teuthology.orchestra.run.smithi100.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-07-28T19:12:37.673 INFO:teuthology.orchestra.run.smithi100.stdout:uid [ unknown] Ceph.com (release key) 2024-07-28T19:12:37.674 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-07-28T19:12:37.674 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-28T19:12:37.674 INFO:teuthology.packaging:ref: None 2024-07-28T19:12:37.674 INFO:teuthology.packaging:tag: None 2024-07-28T19:12:37.675 INFO:teuthology.packaging:branch: quincy 2024-07-28T19:12:37.675 INFO:teuthology.packaging:sha1: 1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:12:37.675 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-07-28T19:12:37.709 INFO:teuthology.orchestra.run.smithi138.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-07-28T19:12:37.709 INFO:teuthology.orchestra.run.smithi138.stdout:uid [ unknown] Ceph.com (release key) 2024-07-28T19:12:37.710 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-07-28T19:12:37.710 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-28T19:12:37.710 INFO:teuthology.packaging:ref: None 2024-07-28T19:12:37.710 INFO:teuthology.packaging:tag: None 2024-07-28T19:12:37.710 INFO:teuthology.packaging:branch: quincy 2024-07-28T19:12:37.710 INFO:teuthology.packaging:sha1: 1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:12:37.710 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-07-28T19:12:37.833 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-07-28T19:12:37.833 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-07-28T19:12:37.852 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-07-28T19:12:37.853 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-07-28T19:12:37.867 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:12:37.868 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-07-28T19:12:37.881 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:12:37.881 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-07-28T19:12:37.895 DEBUG:teuthology.orchestra.run.smithi138:> sudo apt-get update 2024-07-28T19:12:37.917 DEBUG:teuthology.orchestra.run.smithi100:> sudo apt-get update 2024-07-28T19:12:38.037 INFO:teuthology.orchestra.run.smithi138.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-07-28T19:12:38.038 INFO:teuthology.orchestra.run.smithi138.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-07-28T19:12:38.040 INFO:teuthology.orchestra.run.smithi138.stdout:Ign:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-07-28T19:12:38.041 INFO:teuthology.orchestra.run.smithi138.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-07-28T19:12:38.049 INFO:teuthology.orchestra.run.smithi138.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-07-28T19:12:38.061 INFO:teuthology.orchestra.run.smithi100.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-07-28T19:12:38.062 INFO:teuthology.orchestra.run.smithi100.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-07-28T19:12:38.062 INFO:teuthology.orchestra.run.smithi138.stdout:Hit:6 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-07-28T19:12:38.063 INFO:teuthology.orchestra.run.smithi100.stdout:Ign:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-07-28T19:12:38.064 INFO:teuthology.orchestra.run.smithi100.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-07-28T19:12:38.072 INFO:teuthology.orchestra.run.smithi100.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-07-28T19:12:38.185 INFO:teuthology.orchestra.run.smithi138.stdout:Hit:7 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-28T19:12:38.217 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:6 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-07-28T19:12:38.218 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:7 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-07-28T19:12:38.267 INFO:teuthology.orchestra.run.smithi138.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-07-28T19:12:38.290 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-07-28T19:12:38.349 INFO:teuthology.orchestra.run.smithi138.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-07-28T19:12:38.371 INFO:teuthology.orchestra.run.smithi100.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-07-28T19:12:38.492 INFO:teuthology.orchestra.run.smithi138.stdout:Fetched 28.3 kB in 0s (68.2 kB/s) 2024-07-28T19:12:38.592 INFO:teuthology.orchestra.run.smithi100.stdout:Fetched 28.3 kB in 0s (57.6 kB/s) 2024-07-28T19:12:39.449 INFO:teuthology.orchestra.run.smithi138.stdout:Reading package lists... 2024-07-28T19:12:39.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-07-28T19:12:39.527 INFO:teuthology.orchestra.run.smithi138.stdout:Reading package lists... 2024-07-28T19:12:39.607 INFO:teuthology.orchestra.run.smithi100.stdout:Reading package lists... 2024-07-28T19:12:39.624 DEBUG:teuthology.orchestra.run.smithi100:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-07-28T19:12:39.678 INFO:teuthology.orchestra.run.smithi100.stdout:Reading package lists... 2024-07-28T19:12:39.716 INFO:teuthology.orchestra.run.smithi138.stdout:Building dependency tree... 2024-07-28T19:12:39.717 INFO:teuthology.orchestra.run.smithi138.stdout:Reading state information... 2024-07-28T19:12:39.860 INFO:teuthology.orchestra.run.smithi100.stdout:Building dependency tree... 2024-07-28T19:12:39.860 INFO:teuthology.orchestra.run.smithi100.stdout:Reading state information... 2024-07-28T19:12:39.898 INFO:teuthology.orchestra.run.smithi138.stdout:The following packages were automatically installed and are no longer required: 2024-07-28T19:12:39.898 INFO:teuthology.orchestra.run.smithi138.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-28T19:12:39.898 INFO:teuthology.orchestra.run.smithi138.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-28T19:12:39.898 INFO:teuthology.orchestra.run.smithi138.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-28T19:12:39.899 INFO:teuthology.orchestra.run.smithi138.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-07-28T19:12:39.899 INFO:teuthology.orchestra.run.smithi138.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-07-28T19:12:39.899 INFO:teuthology.orchestra.run.smithi138.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-28T19:12:39.899 INFO:teuthology.orchestra.run.smithi138.stdout:The following additional packages will be installed: 2024-07-28T19:12:39.899 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-07-28T19:12:39.900 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-07-28T19:12:39.900 INFO:teuthology.orchestra.run.smithi138.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-07-28T19:12:39.900 INFO:teuthology.orchestra.run.smithi138.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-07-28T19:12:39.901 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout:Suggested packages: 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-07-28T19:12:39.902 INFO:teuthology.orchestra.run.smithi138.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-07-28T19:12:39.903 INFO:teuthology.orchestra.run.smithi138.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-07-28T19:12:39.903 INFO:teuthology.orchestra.run.smithi138.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-07-28T19:12:39.903 INFO:teuthology.orchestra.run.smithi138.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-07-28T19:12:39.903 INFO:teuthology.orchestra.run.smithi138.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-07-28T19:12:39.903 INFO:teuthology.orchestra.run.smithi138.stdout: python3-watchdog 2024-07-28T19:12:39.903 INFO:teuthology.orchestra.run.smithi138.stdout:Recommended packages: 2024-07-28T19:12:39.903 INFO:teuthology.orchestra.run.smithi138.stdout: btrfs-tools 2024-07-28T19:12:40.005 INFO:teuthology.orchestra.run.smithi138.stdout:The following NEW packages will be installed: 2024-07-28T19:12:40.005 INFO:teuthology.orchestra.run.smithi138.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-07-28T19:12:40.005 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-07-28T19:12:40.005 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-07-28T19:12:40.006 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-07-28T19:12:40.006 INFO:teuthology.orchestra.run.smithi138.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-07-28T19:12:40.006 INFO:teuthology.orchestra.run.smithi138.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-07-28T19:12:40.006 INFO:teuthology.orchestra.run.smithi138.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-07-28T19:12:40.006 INFO:teuthology.orchestra.run.smithi138.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-07-28T19:12:40.007 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-07-28T19:12:40.007 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-07-28T19:12:40.007 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-07-28T19:12:40.007 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-07-28T19:12:40.007 INFO:teuthology.orchestra.run.smithi138.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-07-28T19:12:40.008 INFO:teuthology.orchestra.run.smithi138.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-07-28T19:12:40.008 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-07-28T19:12:40.008 INFO:teuthology.orchestra.run.smithi138.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-07-28T19:12:40.008 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-07-28T19:12:40.008 INFO:teuthology.orchestra.run.smithi138.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-07-28T19:12:40.008 INFO:teuthology.orchestra.run.smithi138.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-07-28T19:12:40.009 INFO:teuthology.orchestra.run.smithi138.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-07-28T19:12:40.009 INFO:teuthology.orchestra.run.smithi138.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-07-28T19:12:40.009 INFO:teuthology.orchestra.run.smithi138.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-07-28T19:12:40.009 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-07-28T19:12:40.009 INFO:teuthology.orchestra.run.smithi138.stdout: radosgw rbd-fuse socat xmlstarlet 2024-07-28T19:12:40.042 INFO:teuthology.orchestra.run.smithi100.stdout:The following packages were automatically installed and are no longer required: 2024-07-28T19:12:40.043 INFO:teuthology.orchestra.run.smithi100.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-07-28T19:12:40.043 INFO:teuthology.orchestra.run.smithi100.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-07-28T19:12:40.043 INFO:teuthology.orchestra.run.smithi100.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-07-28T19:12:40.043 INFO:teuthology.orchestra.run.smithi100.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-07-28T19:12:40.043 INFO:teuthology.orchestra.run.smithi100.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-07-28T19:12:40.044 INFO:teuthology.orchestra.run.smithi100.stdout:Use 'sudo apt autoremove' to remove them. 2024-07-28T19:12:40.044 INFO:teuthology.orchestra.run.smithi100.stdout:The following additional packages will be installed: 2024-07-28T19:12:40.044 INFO:teuthology.orchestra.run.smithi100.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-07-28T19:12:40.044 INFO:teuthology.orchestra.run.smithi100.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-07-28T19:12:40.045 INFO:teuthology.orchestra.run.smithi100.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-07-28T19:12:40.045 INFO:teuthology.orchestra.run.smithi100.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-07-28T19:12:40.045 INFO:teuthology.orchestra.run.smithi138.stdout:0 upgraded, 107 newly installed, 0 to remove and 317 not upgraded. 2024-07-28T19:12:40.045 INFO:teuthology.orchestra.run.smithi138.stdout:Need to get 187 MB of archives. 2024-07-28T19:12:40.045 INFO:teuthology.orchestra.run.smithi138.stdout:After this operation, 815 MB of additional disk space will be used. 2024-07-28T19:12:40.046 INFO:teuthology.orchestra.run.smithi138.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-07-28T19:12:40.046 INFO:teuthology.orchestra.run.smithi100.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-07-28T19:12:40.046 INFO:teuthology.orchestra.run.smithi100.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-07-28T19:12:40.046 INFO:teuthology.orchestra.run.smithi100.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-07-28T19:12:40.046 INFO:teuthology.orchestra.run.smithi100.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-07-28T19:12:40.046 INFO:teuthology.orchestra.run.smithi100.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-07-28T19:12:40.046 INFO:teuthology.orchestra.run.smithi100.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout:Suggested packages: 2024-07-28T19:12:40.047 INFO:teuthology.orchestra.run.smithi100.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-07-28T19:12:40.048 INFO:teuthology.orchestra.run.smithi100.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-07-28T19:12:40.048 INFO:teuthology.orchestra.run.smithi100.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-07-28T19:12:40.048 INFO:teuthology.orchestra.run.smithi100.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-07-28T19:12:40.048 INFO:teuthology.orchestra.run.smithi100.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-07-28T19:12:40.048 INFO:teuthology.orchestra.run.smithi100.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-07-28T19:12:40.048 INFO:teuthology.orchestra.run.smithi100.stdout: python3-watchdog 2024-07-28T19:12:40.048 INFO:teuthology.orchestra.run.smithi100.stdout:Recommended packages: 2024-07-28T19:12:40.048 INFO:teuthology.orchestra.run.smithi100.stdout: btrfs-tools 2024-07-28T19:12:40.073 INFO:teuthology.orchestra.run.smithi138.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-07-28T19:12:40.095 INFO:teuthology.orchestra.run.smithi138.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-07-28T19:12:40.099 INFO:teuthology.orchestra.run.smithi138.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-07-28T19:12:40.102 INFO:teuthology.orchestra.run.smithi138.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-07-28T19:12:40.102 INFO:teuthology.orchestra.run.smithi138.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-07-28T19:12:40.104 INFO:teuthology.orchestra.run.smithi138.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-07-28T19:12:40.104 INFO:teuthology.orchestra.run.smithi138.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-07-28T19:12:40.106 INFO:teuthology.orchestra.run.smithi138.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-07-28T19:12:40.146 INFO:teuthology.orchestra.run.smithi100.stdout:The following NEW packages will be installed: 2024-07-28T19:12:40.146 INFO:teuthology.orchestra.run.smithi100.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-07-28T19:12:40.146 INFO:teuthology.orchestra.run.smithi100.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-07-28T19:12:40.146 INFO:teuthology.orchestra.run.smithi100.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-07-28T19:12:40.147 INFO:teuthology.orchestra.run.smithi100.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-07-28T19:12:40.147 INFO:teuthology.orchestra.run.smithi100.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-07-28T19:12:40.147 INFO:teuthology.orchestra.run.smithi100.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-07-28T19:12:40.147 INFO:teuthology.orchestra.run.smithi100.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-07-28T19:12:40.147 INFO:teuthology.orchestra.run.smithi100.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-07-28T19:12:40.148 INFO:teuthology.orchestra.run.smithi100.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-07-28T19:12:40.149 INFO:teuthology.orchestra.run.smithi100.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-07-28T19:12:40.149 INFO:teuthology.orchestra.run.smithi100.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-07-28T19:12:40.149 INFO:teuthology.orchestra.run.smithi100.stdout: radosgw rbd-fuse socat xmlstarlet 2024-07-28T19:12:40.149 INFO:teuthology.orchestra.run.smithi138.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-07-28T19:12:40.150 INFO:teuthology.orchestra.run.smithi138.stdout:Get:11 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-07-28T19:12:40.153 INFO:teuthology.orchestra.run.smithi138.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-07-28T19:12:40.189 INFO:teuthology.orchestra.run.smithi100.stdout:0 upgraded, 107 newly installed, 0 to remove and 317 not upgraded. 2024-07-28T19:12:40.189 INFO:teuthology.orchestra.run.smithi100.stdout:Need to get 187 MB of archives. 2024-07-28T19:12:40.189 INFO:teuthology.orchestra.run.smithi100.stdout:After this operation, 815 MB of additional disk space will be used. 2024-07-28T19:12:40.189 INFO:teuthology.orchestra.run.smithi100.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-07-28T19:12:40.193 INFO:teuthology.orchestra.run.smithi138.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-07-28T19:12:40.218 INFO:teuthology.orchestra.run.smithi100.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-07-28T19:12:40.241 INFO:teuthology.orchestra.run.smithi100.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-07-28T19:12:40.246 INFO:teuthology.orchestra.run.smithi100.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-07-28T19:12:40.248 INFO:teuthology.orchestra.run.smithi100.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-07-28T19:12:40.249 INFO:teuthology.orchestra.run.smithi100.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-07-28T19:12:40.250 INFO:teuthology.orchestra.run.smithi100.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-07-28T19:12:40.250 INFO:teuthology.orchestra.run.smithi100.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-07-28T19:12:40.253 INFO:teuthology.orchestra.run.smithi100.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-07-28T19:12:40.286 INFO:teuthology.orchestra.run.smithi100.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-07-28T19:12:40.287 INFO:teuthology.orchestra.run.smithi100.stdout:Get:11 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-07-28T19:12:40.290 INFO:teuthology.orchestra.run.smithi100.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-07-28T19:12:40.336 INFO:teuthology.orchestra.run.smithi100.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-07-28T19:12:40.451 INFO:teuthology.orchestra.run.smithi138.stdout:Get:14 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-07-28T19:12:40.494 INFO:teuthology.orchestra.run.smithi138.stdout:Get:15 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-07-28T19:12:40.495 INFO:teuthology.orchestra.run.smithi138.stdout:Get:16 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-07-28T19:12:40.496 INFO:teuthology.orchestra.run.smithi138.stdout:Get:17 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-07-28T19:12:40.506 INFO:teuthology.orchestra.run.smithi138.stdout:Get:18 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-07-28T19:12:40.527 INFO:teuthology.orchestra.run.smithi100.stdout:Get:14 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-07-28T19:12:40.535 INFO:teuthology.orchestra.run.smithi138.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-07-28T19:12:40.559 INFO:teuthology.orchestra.run.smithi138.stdout:Get:20 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-07-28T19:12:40.570 INFO:teuthology.orchestra.run.smithi100.stdout:Get:15 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-07-28T19:12:40.619 INFO:teuthology.orchestra.run.smithi100.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-07-28T19:12:40.637 INFO:teuthology.orchestra.run.smithi100.stdout:Get:17 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-07-28T19:12:40.638 INFO:teuthology.orchestra.run.smithi100.stdout:Get:18 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-07-28T19:12:40.648 INFO:teuthology.orchestra.run.smithi100.stdout:Get:19 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-07-28T19:12:40.704 INFO:teuthology.orchestra.run.smithi100.stdout:Get:20 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-07-28T19:12:40.711 INFO:teuthology.orchestra.run.smithi138.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-07-28T19:12:40.720 INFO:teuthology.orchestra.run.smithi138.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-07-28T19:12:40.732 INFO:teuthology.orchestra.run.smithi138.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-07-28T19:12:40.790 INFO:teuthology.orchestra.run.smithi100.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-07-28T19:12:40.798 INFO:teuthology.orchestra.run.smithi100.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-07-28T19:12:40.810 INFO:teuthology.orchestra.run.smithi100.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-07-28T19:12:40.826 INFO:teuthology.orchestra.run.smithi138.stdout:Get:24 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-07-28T19:12:40.900 INFO:teuthology.orchestra.run.smithi138.stdout:Get:25 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-07-28T19:12:40.900 INFO:teuthology.orchestra.run.smithi138.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-07-28T19:12:40.901 INFO:teuthology.orchestra.run.smithi138.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-07-28T19:12:40.905 INFO:teuthology.orchestra.run.smithi138.stdout:Get:28 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-07-28T19:12:40.907 INFO:teuthology.orchestra.run.smithi138.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-07-28T19:12:40.920 INFO:teuthology.orchestra.run.smithi138.stdout:Get:30 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-07-28T19:12:40.920 INFO:teuthology.orchestra.run.smithi138.stdout:Get:31 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-07-28T19:12:40.921 INFO:teuthology.orchestra.run.smithi138.stdout:Get:32 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-07-28T19:12:40.934 INFO:teuthology.orchestra.run.smithi138.stdout:Get:33 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-07-28T19:12:40.965 INFO:teuthology.orchestra.run.smithi100.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-07-28T19:12:40.990 INFO:teuthology.orchestra.run.smithi138.stdout:Get:34 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-07-28T19:12:40.990 INFO:teuthology.orchestra.run.smithi138.stdout:Get:35 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-07-28T19:12:41.022 INFO:teuthology.orchestra.run.smithi138.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-07-28T19:12:41.029 INFO:teuthology.orchestra.run.smithi138.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-07-28T19:12:41.037 INFO:teuthology.orchestra.run.smithi100.stdout:Get:25 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-07-28T19:12:41.037 INFO:teuthology.orchestra.run.smithi100.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-07-28T19:12:41.038 INFO:teuthology.orchestra.run.smithi100.stdout:Get:27 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-07-28T19:12:41.043 INFO:teuthology.orchestra.run.smithi100.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-07-28T19:12:41.043 INFO:teuthology.orchestra.run.smithi100.stdout:Get:29 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-07-28T19:12:41.053 INFO:teuthology.orchestra.run.smithi138.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-07-28T19:12:41.058 INFO:teuthology.orchestra.run.smithi100.stdout:Get:30 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-07-28T19:12:41.059 INFO:teuthology.orchestra.run.smithi100.stdout:Get:31 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-07-28T19:12:41.060 INFO:teuthology.orchestra.run.smithi100.stdout:Get:32 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-07-28T19:12:41.060 INFO:teuthology.orchestra.run.smithi138.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-07-28T19:12:41.062 INFO:teuthology.orchestra.run.smithi138.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-07-28T19:12:41.063 INFO:teuthology.orchestra.run.smithi138.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-07-28T19:12:41.065 INFO:teuthology.orchestra.run.smithi138.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-07-28T19:12:41.067 INFO:teuthology.orchestra.run.smithi138.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-07-28T19:12:41.069 INFO:teuthology.orchestra.run.smithi138.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-07-28T19:12:41.072 INFO:teuthology.orchestra.run.smithi100.stdout:Get:33 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-07-28T19:12:41.127 INFO:teuthology.orchestra.run.smithi100.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-07-28T19:12:41.128 INFO:teuthology.orchestra.run.smithi100.stdout:Get:35 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-07-28T19:12:41.128 INFO:teuthology.orchestra.run.smithi100.stdout:Get:36 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-07-28T19:12:41.133 INFO:teuthology.orchestra.run.smithi100.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-07-28T19:12:41.149 INFO:teuthology.orchestra.run.smithi138.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-07-28T19:12:41.149 INFO:teuthology.orchestra.run.smithi138.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-07-28T19:12:41.150 INFO:teuthology.orchestra.run.smithi138.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-07-28T19:12:41.151 INFO:teuthology.orchestra.run.smithi138.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-07-28T19:12:41.151 INFO:teuthology.orchestra.run.smithi138.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-07-28T19:12:41.152 INFO:teuthology.orchestra.run.smithi138.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-07-28T19:12:41.153 INFO:teuthology.orchestra.run.smithi100.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-07-28T19:12:41.157 INFO:teuthology.orchestra.run.smithi138.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-07-28T19:12:41.158 INFO:teuthology.orchestra.run.smithi138.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-07-28T19:12:41.159 INFO:teuthology.orchestra.run.smithi100.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-07-28T19:12:41.161 INFO:teuthology.orchestra.run.smithi100.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-07-28T19:12:41.162 INFO:teuthology.orchestra.run.smithi100.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-07-28T19:12:41.163 INFO:teuthology.orchestra.run.smithi100.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-07-28T19:12:41.203 INFO:teuthology.orchestra.run.smithi100.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-07-28T19:12:41.205 INFO:teuthology.orchestra.run.smithi100.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-07-28T19:12:41.231 INFO:teuthology.orchestra.run.smithi138.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-07-28T19:12:41.231 INFO:teuthology.orchestra.run.smithi138.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-07-28T19:12:41.285 INFO:teuthology.orchestra.run.smithi100.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-07-28T19:12:41.286 INFO:teuthology.orchestra.run.smithi100.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-07-28T19:12:41.286 INFO:teuthology.orchestra.run.smithi100.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-07-28T19:12:41.287 INFO:teuthology.orchestra.run.smithi100.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-07-28T19:12:41.288 INFO:teuthology.orchestra.run.smithi100.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-07-28T19:12:41.289 INFO:teuthology.orchestra.run.smithi100.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-07-28T19:12:41.293 INFO:teuthology.orchestra.run.smithi100.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-07-28T19:12:41.295 INFO:teuthology.orchestra.run.smithi100.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-07-28T19:12:41.312 INFO:teuthology.orchestra.run.smithi138.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-07-28T19:12:41.313 INFO:teuthology.orchestra.run.smithi138.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-07-28T19:12:41.313 INFO:teuthology.orchestra.run.smithi138.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-07-28T19:12:41.316 INFO:teuthology.orchestra.run.smithi138.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-07-28T19:12:41.318 INFO:teuthology.orchestra.run.smithi138.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-07-28T19:12:41.319 INFO:teuthology.orchestra.run.smithi138.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-07-28T19:12:41.367 INFO:teuthology.orchestra.run.smithi100.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-07-28T19:12:41.368 INFO:teuthology.orchestra.run.smithi100.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-07-28T19:12:41.402 INFO:teuthology.orchestra.run.smithi138.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-07-28T19:12:41.403 INFO:teuthology.orchestra.run.smithi138.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-07-28T19:12:41.449 INFO:teuthology.orchestra.run.smithi100.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-07-28T19:12:41.450 INFO:teuthology.orchestra.run.smithi100.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-07-28T19:12:41.451 INFO:teuthology.orchestra.run.smithi100.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-07-28T19:12:41.453 INFO:teuthology.orchestra.run.smithi100.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-07-28T19:12:41.478 INFO:teuthology.orchestra.run.smithi138.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-07-28T19:12:41.539 INFO:teuthology.orchestra.run.smithi100.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-07-28T19:12:41.540 INFO:teuthology.orchestra.run.smithi100.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-07-28T19:12:41.587 INFO:teuthology.orchestra.run.smithi138.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-07-28T19:12:41.588 INFO:teuthology.orchestra.run.smithi138.stdout:Get:65 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-07-28T19:12:41.589 INFO:teuthology.orchestra.run.smithi138.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-07-28T19:12:41.590 INFO:teuthology.orchestra.run.smithi138.stdout:Get:67 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-07-28T19:12:41.591 INFO:teuthology.orchestra.run.smithi138.stdout:Get:68 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-07-28T19:12:41.593 INFO:teuthology.orchestra.run.smithi138.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-07-28T19:12:41.594 INFO:teuthology.orchestra.run.smithi138.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-07-28T19:12:41.597 INFO:teuthology.orchestra.run.smithi138.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-07-28T19:12:41.599 INFO:teuthology.orchestra.run.smithi138.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-07-28T19:12:41.602 INFO:teuthology.orchestra.run.smithi100.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-07-28T19:12:41.602 INFO:teuthology.orchestra.run.smithi100.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-07-28T19:12:41.603 INFO:teuthology.orchestra.run.smithi138.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-07-28T19:12:41.693 INFO:teuthology.orchestra.run.smithi100.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-07-28T19:12:41.693 INFO:teuthology.orchestra.run.smithi100.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-07-28T19:12:41.756 INFO:teuthology.orchestra.run.smithi100.stdout:Get:65 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-07-28T19:12:41.756 INFO:teuthology.orchestra.run.smithi100.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-07-28T19:12:41.758 INFO:teuthology.orchestra.run.smithi100.stdout:Get:67 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-07-28T19:12:41.759 INFO:teuthology.orchestra.run.smithi100.stdout:Get:68 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-07-28T19:12:41.759 INFO:teuthology.orchestra.run.smithi100.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-07-28T19:12:41.761 INFO:teuthology.orchestra.run.smithi100.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-07-28T19:12:41.764 INFO:teuthology.orchestra.run.smithi100.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-07-28T19:12:41.765 INFO:teuthology.orchestra.run.smithi100.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-07-28T19:12:41.770 INFO:teuthology.orchestra.run.smithi100.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-07-28T19:12:41.799 INFO:teuthology.orchestra.run.smithi138.stdout:Get:74 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-07-28T19:12:41.800 INFO:teuthology.orchestra.run.smithi138.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-07-28T19:12:41.802 INFO:teuthology.orchestra.run.smithi138.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-07-28T19:12:41.805 INFO:teuthology.orchestra.run.smithi138.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-07-28T19:12:41.806 INFO:teuthology.orchestra.run.smithi138.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-07-28T19:12:41.808 INFO:teuthology.orchestra.run.smithi138.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-07-28T19:12:41.851 INFO:teuthology.orchestra.run.smithi138.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-07-28T19:12:41.854 INFO:teuthology.orchestra.run.smithi138.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-07-28T19:12:41.855 INFO:teuthology.orchestra.run.smithi138.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-07-28T19:12:41.919 INFO:teuthology.orchestra.run.smithi138.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-07-28T19:12:41.919 INFO:teuthology.orchestra.run.smithi138.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-07-28T19:12:41.965 INFO:teuthology.orchestra.run.smithi100.stdout:Get:74 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-07-28T19:12:41.965 INFO:teuthology.orchestra.run.smithi100.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-07-28T19:12:41.967 INFO:teuthology.orchestra.run.smithi100.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-07-28T19:12:41.968 INFO:teuthology.orchestra.run.smithi138.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-07-28T19:12:41.969 INFO:teuthology.orchestra.run.smithi138.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-07-28T19:12:41.969 INFO:teuthology.orchestra.run.smithi100.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-07-28T19:12:41.970 INFO:teuthology.orchestra.run.smithi138.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-07-28T19:12:41.970 INFO:teuthology.orchestra.run.smithi100.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-07-28T19:12:41.971 INFO:teuthology.orchestra.run.smithi138.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-07-28T19:12:41.972 INFO:teuthology.orchestra.run.smithi138.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-07-28T19:12:41.972 INFO:teuthology.orchestra.run.smithi100.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-07-28T19:12:41.973 INFO:teuthology.orchestra.run.smithi138.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-07-28T19:12:41.980 INFO:teuthology.orchestra.run.smithi138.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-07-28T19:12:42.010 INFO:teuthology.orchestra.run.smithi100.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-07-28T19:12:42.013 INFO:teuthology.orchestra.run.smithi100.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-07-28T19:12:42.014 INFO:teuthology.orchestra.run.smithi100.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-07-28T19:12:42.049 INFO:teuthology.orchestra.run.smithi138.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-07-28T19:12:42.052 INFO:teuthology.orchestra.run.smithi138.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-07-28T19:12:42.054 INFO:teuthology.orchestra.run.smithi138.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-07-28T19:12:42.064 INFO:teuthology.orchestra.run.smithi100.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-07-28T19:12:42.065 INFO:teuthology.orchestra.run.smithi100.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-07-28T19:12:42.131 INFO:teuthology.orchestra.run.smithi138.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-07-28T19:12:42.132 INFO:teuthology.orchestra.run.smithi100.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-07-28T19:12:42.133 INFO:teuthology.orchestra.run.smithi100.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-07-28T19:12:42.134 INFO:teuthology.orchestra.run.smithi100.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-07-28T19:12:42.136 INFO:teuthology.orchestra.run.smithi100.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-07-28T19:12:42.136 INFO:teuthology.orchestra.run.smithi100.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-07-28T19:12:42.137 INFO:teuthology.orchestra.run.smithi138.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.2 [473 kB] 2024-07-28T19:12:42.138 INFO:teuthology.orchestra.run.smithi100.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-07-28T19:12:42.146 INFO:teuthology.orchestra.run.smithi100.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-07-28T19:12:42.148 INFO:teuthology.orchestra.run.smithi138.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-07-28T19:12:42.148 INFO:teuthology.orchestra.run.smithi138.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-07-28T19:12:42.150 INFO:teuthology.orchestra.run.smithi138.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-07-28T19:12:42.151 INFO:teuthology.orchestra.run.smithi138.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-07-28T19:12:42.154 INFO:teuthology.orchestra.run.smithi138.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-07-28T19:12:42.212 INFO:teuthology.orchestra.run.smithi138.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-07-28T19:12:42.214 INFO:teuthology.orchestra.run.smithi100.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-07-28T19:12:42.218 INFO:teuthology.orchestra.run.smithi100.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-07-28T19:12:42.219 INFO:teuthology.orchestra.run.smithi100.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-07-28T19:12:42.229 INFO:teuthology.orchestra.run.smithi138.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-07-28T19:12:42.229 INFO:teuthology.orchestra.run.smithi138.stdout:Get:104 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-07-28T19:12:42.294 INFO:teuthology.orchestra.run.smithi138.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-07-28T19:12:42.295 INFO:teuthology.orchestra.run.smithi100.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-07-28T19:12:42.298 INFO:teuthology.orchestra.run.smithi138.stdout:Get:106 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-07-28T19:12:42.300 INFO:teuthology.orchestra.run.smithi138.stdout:Get:107 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-07-28T19:12:42.302 INFO:teuthology.orchestra.run.smithi100.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.2 [473 kB] 2024-07-28T19:12:42.314 INFO:teuthology.orchestra.run.smithi100.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-07-28T19:12:42.314 INFO:teuthology.orchestra.run.smithi100.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-07-28T19:12:42.316 INFO:teuthology.orchestra.run.smithi100.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-07-28T19:12:42.317 INFO:teuthology.orchestra.run.smithi100.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-07-28T19:12:42.320 INFO:teuthology.orchestra.run.smithi100.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-07-28T19:12:42.377 INFO:teuthology.orchestra.run.smithi100.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-07-28T19:12:42.394 INFO:teuthology.orchestra.run.smithi100.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-07-28T19:12:42.394 INFO:teuthology.orchestra.run.smithi100.stdout:Get:104 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-07-28T19:12:42.459 INFO:teuthology.orchestra.run.smithi100.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-07-28T19:12:42.464 INFO:teuthology.orchestra.run.smithi100.stdout:Get:106 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-07-28T19:12:42.465 INFO:teuthology.orchestra.run.smithi100.stdout:Get:107 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-07-28T19:12:42.860 INFO:teuthology.orchestra.run.smithi138.stdout:Fetched 187 MB in 2s (80.1 MB/s) 2024-07-28T19:12:42.984 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libfmt8:amd64. 2024-07-28T19:12:43.021 INFO:teuthology.orchestra.run.smithi100.stdout:Fetched 187 MB in 2s (79.4 MB/s) 2024-07-28T19:12:43.041 INFO:teuthology.orchestra.run.smithi138.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165992 files and directories currently installed.) 2024-07-28T19:12:43.046 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-07-28T19:12:43.117 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-07-28T19:12:43.125 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libfmt8:amd64. 2024-07-28T19:12:43.215 INFO:teuthology.orchestra.run.smithi100.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165992 files and directories currently installed.) 2024-07-28T19:12:43.220 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-07-28T19:12:43.300 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-07-28T19:12:43.437 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-07-28T19:12:43.463 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-07-28T19:12:43.504 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-07-28T19:12:43.644 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-07-28T19:12:43.670 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-07-28T19:12:43.711 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-07-28T19:12:43.832 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package librados2. 2024-07-28T19:12:43.858 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:43.899 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-07-28T19:12:44.039 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package librados2. 2024-07-28T19:12:44.065 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:44.105 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-07-28T19:12:44.360 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package librbd1. 2024-07-28T19:12:44.386 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:44.426 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-07-28T19:12:44.550 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package librbd1. 2024-07-28T19:12:44.576 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:44.616 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-07-28T19:12:44.854 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libcephfs2. 2024-07-28T19:12:44.880 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:44.921 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-07-28T19:12:45.053 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libcephfs2. 2024-07-28T19:12:45.079 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:45.119 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-07-28T19:12:45.207 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-rados. 2024-07-28T19:12:45.233 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:45.273 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-07-28T19:12:45.372 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-rados. 2024-07-28T19:12:45.386 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:45.421 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-07-28T19:12:45.551 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-07-28T19:12:45.578 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-07-28T19:12:45.609 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-07-28T19:12:45.749 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-07-28T19:12:45.775 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-07-28T19:12:45.816 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-07-28T19:12:45.845 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-cephfs. 2024-07-28T19:12:45.862 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:45.928 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-07-28T19:12:46.051 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-cephfs. 2024-07-28T19:12:46.067 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:46.110 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-07-28T19:12:46.239 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-ceph-common. 2024-07-28T19:12:46.266 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-07-28T19:12:46.306 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-07-28T19:12:46.421 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-ceph-common. 2024-07-28T19:12:46.447 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-07-28T19:12:46.488 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-07-28T19:12:46.617 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-wcwidth. 2024-07-28T19:12:46.644 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-07-28T19:12:46.684 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-07-28T19:12:46.799 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-wcwidth. 2024-07-28T19:12:46.825 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-07-28T19:12:46.890 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-07-28T19:12:46.970 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-prettytable. 2024-07-28T19:12:46.997 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-07-28T19:12:47.037 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-07-28T19:12:47.210 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-prettytable. 2024-07-28T19:12:47.236 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-07-28T19:12:47.264 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-rbd. 2024-07-28T19:12:47.279 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:47.285 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-07-28T19:12:47.322 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-07-28T19:12:47.529 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-rbd. 2024-07-28T19:12:47.544 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:47.579 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-07-28T19:12:47.676 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-07-28T19:12:47.701 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-07-28T19:12:47.742 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-07-28T19:12:47.907 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-07-28T19:12:47.933 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-07-28T19:12:47.973 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-07-28T19:12:48.095 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-07-28T19:12:48.121 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-07-28T19:12:48.161 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-07-28T19:12:48.351 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-07-28T19:12:48.377 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-07-28T19:12:48.418 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-07-28T19:12:48.448 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-07-28T19:12:48.463 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-07-28T19:12:48.506 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-07-28T19:12:48.729 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-07-28T19:12:48.755 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-07-28T19:12:48.796 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-07-28T19:12:48.935 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-07-28T19:12:48.961 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-28T19:12:49.010 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:12:49.216 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-07-28T19:12:49.242 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-28T19:12:49.290 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:12:49.404 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-07-28T19:12:49.430 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-28T19:12:49.471 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:12:49.703 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-07-28T19:12:49.728 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-28T19:12:49.769 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:12:49.782 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-07-28T19:12:49.808 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-28T19:12:49.841 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:12:50.089 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-07-28T19:12:50.104 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-07-28T19:12:50.139 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:12:50.202 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-07-28T19:12:50.228 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-07-28T19:12:50.269 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-07-28T19:12:50.492 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-07-28T19:12:50.518 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-07-28T19:12:50.558 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-07-28T19:12:50.588 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package librgw2. 2024-07-28T19:12:50.614 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:50.655 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-07-28T19:12:50.895 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package librgw2. 2024-07-28T19:12:50.921 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:50.961 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-07-28T19:12:51.115 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-rgw. 2024-07-28T19:12:51.130 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:51.165 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-07-28T19:12:51.438 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-rgw. 2024-07-28T19:12:51.451 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package liboath0:amd64. 2024-07-28T19:12:51.453 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:51.478 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-07-28T19:12:51.497 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-07-28T19:12:51.518 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-07-28T19:12:51.783 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package liboath0:amd64. 2024-07-28T19:12:51.798 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-07-28T19:12:51.829 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libradosstriper1. 2024-07-28T19:12:51.833 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-07-28T19:12:51.855 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:51.896 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-07-28T19:12:52.136 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libradosstriper1. 2024-07-28T19:12:52.162 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:52.202 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-07-28T19:12:52.299 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-common. 2024-07-28T19:12:52.325 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:52.366 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-07-28T19:12:52.631 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-common. 2024-07-28T19:12:52.656 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:52.697 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-07-28T19:12:53.796 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-base. 2024-07-28T19:12:53.822 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:53.872 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-07-28T19:12:54.119 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-base. 2024-07-28T19:12:54.146 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:12:54.187 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-07-28T19:12:54.542 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-openssl. 2024-07-28T19:12:54.569 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-07-28T19:12:54.609 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-07-28T19:12:54.824 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-openssl. 2024-07-28T19:12:54.850 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-07-28T19:12:54.878 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-07-28T19:12:54.890 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-07-28T19:12:54.893 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-07-28T19:12:54.936 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-07-28T19:12:55.168 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-07-28T19:12:55.183 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-07-28T19:12:55.226 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-07-28T19:12:55.300 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-bcrypt. 2024-07-28T19:12:55.326 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-07-28T19:12:55.366 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-07-28T19:12:55.548 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-bcrypt. 2024-07-28T19:12:55.562 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-07-28T19:12:55.597 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-07-28T19:12:55.619 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-07-28T19:12:55.645 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-07-28T19:12:55.685 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-07-28T19:12:55.892 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-07-28T19:12:55.918 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-07-28T19:12:55.958 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-07-28T19:12:55.988 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-cheroot. 2024-07-28T19:12:56.014 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-07-28T19:12:56.054 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-07-28T19:12:56.261 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-cheroot. 2024-07-28T19:12:56.288 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-07-28T19:12:56.328 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-07-28T19:12:56.366 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-07-28T19:12:56.392 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-07-28T19:12:56.432 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-07-28T19:12:56.664 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-07-28T19:12:56.691 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-07-28T19:12:56.730 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-07-28T19:12:56.735 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-jaraco.text. 2024-07-28T19:12:56.762 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-07-28T19:12:56.802 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-07-28T19:12:57.025 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-jaraco.text. 2024-07-28T19:12:57.052 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-07-28T19:12:57.063 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-07-28T19:12:57.083 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-07-28T19:12:57.090 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-07-28T19:12:57.121 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-07-28T19:12:57.361 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-07-28T19:12:57.388 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-07-28T19:12:57.428 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-07-28T19:12:57.433 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-tempora. 2024-07-28T19:12:57.459 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-07-28T19:12:57.499 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-07-28T19:12:57.731 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-tempora. 2024-07-28T19:12:57.757 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-07-28T19:12:57.797 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-07-28T19:12:58.044 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-portend. 2024-07-28T19:12:58.070 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-07-28T19:12:58.100 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-portend. 2024-07-28T19:12:58.110 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-07-28T19:12:58.127 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-07-28T19:12:58.166 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-07-28T19:12:58.428 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-07-28T19:12:58.438 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-07-28T19:12:58.443 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-07-28T19:12:58.464 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-07-28T19:12:58.486 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-07-28T19:12:58.504 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-07-28T19:12:58.776 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-cherrypy3. 2024-07-28T19:12:58.803 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-07-28T19:12:58.843 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-07-28T19:12:59.031 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-cherrypy3. 2024-07-28T19:12:59.057 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-07-28T19:12:59.089 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-07-28T19:12:59.197 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-natsort. 2024-07-28T19:12:59.223 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-07-28T19:12:59.255 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-07-28T19:12:59.434 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-natsort. 2024-07-28T19:12:59.461 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-07-28T19:12:59.501 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-07-28T19:12:59.551 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-logutils. 2024-07-28T19:12:59.577 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-07-28T19:12:59.617 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-07-28T19:12:59.812 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-logutils. 2024-07-28T19:12:59.839 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-07-28T19:12:59.879 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-07-28T19:12:59.921 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-markupsafe. 2024-07-28T19:12:59.947 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-07-28T19:12:59.987 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-07-28T19:13:00.182 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-markupsafe. 2024-07-28T19:13:00.208 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-07-28T19:13:00.248 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-07-28T19:13:00.274 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-mako. 2024-07-28T19:13:00.300 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-07-28T19:13:00.340 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-07-28T19:13:00.518 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-mako. 2024-07-28T19:13:00.544 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-07-28T19:13:00.584 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-07-28T19:13:00.628 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-simplegeneric. 2024-07-28T19:13:00.643 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-07-28T19:13:00.678 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-07-28T19:13:00.904 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-simplegeneric. 2024-07-28T19:13:00.931 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-07-28T19:13:00.971 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-07-28T19:13:00.973 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-singledispatch. 2024-07-28T19:13:00.999 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-07-28T19:13:01.039 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-07-28T19:13:01.257 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-singledispatch. 2024-07-28T19:13:01.283 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-07-28T19:13:01.323 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-07-28T19:13:01.359 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-webob. 2024-07-28T19:13:01.386 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-07-28T19:13:01.426 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-07-28T19:13:01.593 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-webob. 2024-07-28T19:13:01.619 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-07-28T19:13:01.659 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-07-28T19:13:01.779 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-waitress. 2024-07-28T19:13:01.806 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-07-28T19:13:01.849 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-07-28T19:13:01.962 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-waitress. 2024-07-28T19:13:01.988 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-07-28T19:13:02.024 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-07-28T19:13:02.150 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-tempita. 2024-07-28T19:13:02.176 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-07-28T19:13:02.208 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-07-28T19:13:02.282 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-tempita. 2024-07-28T19:13:02.308 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-07-28T19:13:02.348 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-07-28T19:13:02.528 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-paste. 2024-07-28T19:13:02.554 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-07-28T19:13:02.594 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-07-28T19:13:02.651 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-paste. 2024-07-28T19:13:02.677 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-07-28T19:13:02.717 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-07-28T19:13:02.890 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-07-28T19:13:02.905 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-07-28T19:13:02.940 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-07-28T19:13:02.995 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-07-28T19:13:03.011 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-07-28T19:13:03.045 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-07-28T19:13:03.227 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pastedeploy. 2024-07-28T19:13:03.253 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-07-28T19:13:03.293 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-07-28T19:13:03.340 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pastedeploy. 2024-07-28T19:13:03.366 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-07-28T19:13:03.406 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-07-28T19:13:03.597 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-webtest. 2024-07-28T19:13:03.623 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-07-28T19:13:03.663 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-07-28T19:13:03.735 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-webtest. 2024-07-28T19:13:03.761 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-07-28T19:13:03.801 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-07-28T19:13:03.975 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pecan. 2024-07-28T19:13:04.001 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-07-28T19:13:04.042 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-07-28T19:13:04.121 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pecan. 2024-07-28T19:13:04.147 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-07-28T19:13:04.187 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-07-28T19:13:04.381 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-certifi. 2024-07-28T19:13:04.407 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-07-28T19:13:04.447 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-07-28T19:13:04.542 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-certifi. 2024-07-28T19:13:04.568 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-07-28T19:13:04.608 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-07-28T19:13:04.751 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-idna. 2024-07-28T19:13:04.778 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-07-28T19:13:04.817 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-07-28T19:13:04.895 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-idna. 2024-07-28T19:13:04.921 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-07-28T19:13:04.961 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-07-28T19:13:05.137 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-urllib3. 2024-07-28T19:13:05.164 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-07-28T19:13:05.204 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-07-28T19:13:05.281 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-urllib3. 2024-07-28T19:13:05.307 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-07-28T19:13:05.347 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-07-28T19:13:05.524 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-requests. 2024-07-28T19:13:05.550 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-07-28T19:13:05.591 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-07-28T19:13:05.642 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-requests. 2024-07-28T19:13:05.669 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-07-28T19:13:05.700 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-07-28T19:13:05.886 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-werkzeug. 2024-07-28T19:13:05.913 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-07-28T19:13:05.953 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-07-28T19:13:06.012 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-werkzeug. 2024-07-28T19:13:06.038 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-07-28T19:13:06.078 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-07-28T19:13:06.298 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-07-28T19:13:06.324 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:06.364 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-07-28T19:13:06.415 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-07-28T19:13:06.441 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:06.481 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-07-28T19:13:06.710 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-mgr. 2024-07-28T19:13:06.736 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:06.784 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-mgr. 2024-07-28T19:13:06.785 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-07-28T19:13:06.811 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:06.859 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-07-28T19:13:07.138 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-mon. 2024-07-28T19:13:07.165 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:07.205 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-07-28T19:13:07.229 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-mon. 2024-07-28T19:13:07.255 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:07.295 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-07-28T19:13:08.059 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-osd. 2024-07-28T19:13:08.085 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:08.125 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-07-28T19:13:08.141 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-osd. 2024-07-28T19:13:08.167 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:08.207 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-07-28T19:13:09.760 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph. 2024-07-28T19:13:09.763 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph. 2024-07-28T19:13:09.778 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:09.786 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:09.818 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-07-28T19:13:09.821 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-07-28T19:13:10.096 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-fuse. 2024-07-28T19:13:10.123 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:10.141 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-fuse. 2024-07-28T19:13:10.167 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:10.179 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-07-28T19:13:10.207 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-07-28T19:13:10.516 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-mds. 2024-07-28T19:13:10.543 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:10.582 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-07-28T19:13:10.603 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-mds. 2024-07-28T19:13:10.629 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:10.669 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-07-28T19:13:10.977 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package cephadm. 2024-07-28T19:13:11.003 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:11.040 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package cephadm. 2024-07-28T19:13:11.043 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-07-28T19:13:11.066 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:11.106 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-07-28T19:13:11.321 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python-babel-localedata. 2024-07-28T19:13:11.336 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-07-28T19:13:11.368 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python-babel-localedata. 2024-07-28T19:13:11.371 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-07-28T19:13:11.383 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-07-28T19:13:11.435 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-07-28T19:13:12.360 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-babel. 2024-07-28T19:13:12.386 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-07-28T19:13:12.426 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-07-28T19:13:12.441 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-babel. 2024-07-28T19:13:12.468 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-07-28T19:13:12.507 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-07-28T19:13:12.715 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-jinja2. 2024-07-28T19:13:12.741 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-07-28T19:13:12.781 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-07-28T19:13:12.836 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-jinja2. 2024-07-28T19:13:12.863 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-07-28T19:13:12.903 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-07-28T19:13:13.118 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-07-28T19:13:13.144 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:13.184 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-07-28T19:13:13.265 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-07-28T19:13:13.291 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:13.490 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-07-28T19:13:14.038 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-repoze.lru. 2024-07-28T19:13:14.065 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-07-28T19:13:14.476 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-07-28T19:13:14.694 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-repoze.lru. 2024-07-28T19:13:14.720 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-07-28T19:13:14.760 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-07-28T19:13:14.832 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-routes. 2024-07-28T19:13:14.848 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-07-28T19:13:14.882 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-07-28T19:13:15.039 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-routes. 2024-07-28T19:13:15.066 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-07-28T19:13:15.105 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-07-28T19:13:15.177 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-07-28T19:13:15.204 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:15.235 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-07-28T19:13:15.384 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-07-28T19:13:15.411 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:15.451 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-07-28T19:13:15.697 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-07-28T19:13:15.713 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-07-28T19:13:15.756 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-07-28T19:13:15.872 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-07-28T19:13:15.887 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-07-28T19:13:15.930 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-07-28T19:13:16.243 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-joblib. 2024-07-28T19:13:16.269 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-07-28T19:13:16.309 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-07-28T19:13:16.409 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-joblib. 2024-07-28T19:13:16.436 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-07-28T19:13:16.468 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-07-28T19:13:16.680 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-07-28T19:13:16.706 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-07-28T19:13:16.754 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-07-28T19:13:16.830 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-07-28T19:13:16.857 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-07-28T19:13:16.913 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-07-28T19:13:17.082 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-sklearn. 2024-07-28T19:13:17.121 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-07-28T19:13:17.149 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-07-28T19:13:17.225 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-sklearn. 2024-07-28T19:13:17.252 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-07-28T19:13:17.291 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-07-28T19:13:17.900 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-07-28T19:13:17.927 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:17.943 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-07-28T19:13:17.966 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-07-28T19:13:17.971 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:18.019 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-07-28T19:13:18.911 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-cachetools. 2024-07-28T19:13:18.938 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-07-28T19:13:18.969 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-07-28T19:13:19.006 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-cachetools. 2024-07-28T19:13:19.032 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-07-28T19:13:19.072 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-07-28T19:13:19.280 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pyasn1. 2024-07-28T19:13:19.307 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-07-28T19:13:19.347 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-07-28T19:13:19.379 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pyasn1. 2024-07-28T19:13:19.402 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-07-28T19:13:19.442 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-07-28T19:13:19.709 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-07-28T19:13:19.712 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-07-28T19:13:19.728 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-07-28T19:13:19.736 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-07-28T19:13:19.762 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-07-28T19:13:19.775 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-07-28T19:13:20.024 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-rsa. 2024-07-28T19:13:20.051 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-07-28T19:13:20.082 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-rsa (4.8-1) ... 2024-07-28T19:13:20.086 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-rsa. 2024-07-28T19:13:20.113 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-07-28T19:13:20.178 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-rsa (4.8-1) ... 2024-07-28T19:13:20.404 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-google-auth. 2024-07-28T19:13:20.430 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-07-28T19:13:20.465 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-google-auth. 2024-07-28T19:13:20.470 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-07-28T19:13:20.492 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-07-28T19:13:20.523 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-07-28T19:13:20.790 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-07-28T19:13:20.793 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-07-28T19:13:20.817 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-07-28T19:13:20.820 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-07-28T19:13:20.851 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-07-28T19:13:20.856 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-07-28T19:13:21.129 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-websocket. 2024-07-28T19:13:21.151 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-websocket. 2024-07-28T19:13:21.156 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-07-28T19:13:21.178 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-07-28T19:13:21.195 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-07-28T19:13:21.218 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-07-28T19:13:21.600 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-kubernetes. 2024-07-28T19:13:21.624 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-kubernetes. 2024-07-28T19:13:21.627 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-07-28T19:13:21.651 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-07-28T19:13:21.667 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-07-28T19:13:21.690 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-07-28T19:13:22.322 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-07-28T19:13:22.339 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-07-28T19:13:22.350 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:22.355 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:22.388 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-07-28T19:13:22.397 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-07-28T19:13:22.675 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libonig5:amd64. 2024-07-28T19:13:22.702 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-07-28T19:13:22.717 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libonig5:amd64. 2024-07-28T19:13:22.741 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-07-28T19:13:22.744 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-07-28T19:13:22.783 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-07-28T19:13:23.087 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libjq1:amd64. 2024-07-28T19:13:23.104 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libjq1:amd64. 2024-07-28T19:13:23.114 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-07-28T19:13:23.131 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-07-28T19:13:23.153 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-07-28T19:13:23.170 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-07-28T19:13:23.415 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package jq. 2024-07-28T19:13:23.442 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-07-28T19:13:23.468 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package jq. 2024-07-28T19:13:23.481 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-07-28T19:13:23.496 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-07-28T19:13:23.539 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-07-28T19:13:23.684 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package socat. 2024-07-28T19:13:23.709 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-07-28T19:13:23.751 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-07-28T19:13:23.792 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package socat. 2024-07-28T19:13:23.807 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-07-28T19:13:23.850 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-07-28T19:13:24.014 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package xmlstarlet. 2024-07-28T19:13:24.029 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-07-28T19:13:24.071 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-07-28T19:13:24.170 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package xmlstarlet. 2024-07-28T19:13:24.186 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-07-28T19:13:24.228 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-07-28T19:13:24.418 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-test. 2024-07-28T19:13:24.434 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:24.468 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-07-28T19:13:24.583 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-test. 2024-07-28T19:13:24.610 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:24.649 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-07-28T19:13:26.923 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package ceph-volume. 2024-07-28T19:13:26.950 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:26.980 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-07-28T19:13:27.094 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package ceph-volume. 2024-07-28T19:13:27.110 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-07-28T19:13:27.152 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-07-28T19:13:27.328 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package libcephfs-dev. 2024-07-28T19:13:27.344 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:27.386 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-07-28T19:13:27.499 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package libcephfs-dev. 2024-07-28T19:13:27.526 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:27.565 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-07-28T19:13:27.698 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package nvme-cli. 2024-07-28T19:13:27.726 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.2_amd64.deb ... 2024-07-28T19:13:27.756 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking nvme-cli (1.16-3ubuntu0.2) ... 2024-07-28T19:13:27.893 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package nvme-cli. 2024-07-28T19:13:27.921 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.2_amd64.deb ... 2024-07-28T19:13:27.951 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking nvme-cli (1.16-3ubuntu0.2) ... 2024-07-28T19:13:28.138 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-iniconfig. 2024-07-28T19:13:28.166 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-07-28T19:13:28.205 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-07-28T19:13:28.341 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-iniconfig. 2024-07-28T19:13:28.368 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-07-28T19:13:28.407 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-07-28T19:13:28.508 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pastescript. 2024-07-28T19:13:28.535 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-07-28T19:13:28.574 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-07-28T19:13:28.710 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pastescript. 2024-07-28T19:13:28.737 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-07-28T19:13:28.776 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-07-28T19:13:29.072 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pluggy. 2024-07-28T19:13:29.099 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-07-28T19:13:29.130 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-07-28T19:13:29.162 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pluggy. 2024-07-28T19:13:29.189 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-07-28T19:13:29.229 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-07-28T19:13:29.433 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-psutil. 2024-07-28T19:13:29.460 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-07-28T19:13:29.499 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-07-28T19:13:29.507 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-psutil. 2024-07-28T19:13:29.535 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-07-28T19:13:29.565 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-07-28T19:13:29.836 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-py. 2024-07-28T19:13:29.852 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-07-28T19:13:29.894 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-py (1.10.0-1) ... 2024-07-28T19:13:29.919 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-py. 2024-07-28T19:13:29.946 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-07-28T19:13:29.985 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-py (1.10.0-1) ... 2024-07-28T19:13:30.239 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pygments. 2024-07-28T19:13:30.266 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-07-28T19:13:30.305 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-07-28T19:13:30.315 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pygments. 2024-07-28T19:13:30.342 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-07-28T19:13:30.381 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-07-28T19:13:30.768 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pyinotify. 2024-07-28T19:13:30.795 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-07-28T19:13:30.834 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-07-28T19:13:30.869 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pyinotify. 2024-07-28T19:13:30.897 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-07-28T19:13:30.935 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-07-28T19:13:31.112 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-toml. 2024-07-28T19:13:31.139 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-07-28T19:13:31.170 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-07-28T19:13:31.247 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-toml. 2024-07-28T19:13:31.263 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-07-28T19:13:31.297 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-07-28T19:13:31.448 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-pytest. 2024-07-28T19:13:31.476 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-07-28T19:13:31.515 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-07-28T19:13:31.576 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-pytest. 2024-07-28T19:13:31.603 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-07-28T19:13:31.642 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-07-28T19:13:31.843 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package python3-simplejson. 2024-07-28T19:13:31.871 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-07-28T19:13:31.901 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-07-28T19:13:31.955 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package python3-simplejson. 2024-07-28T19:13:31.982 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-07-28T19:13:32.021 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-07-28T19:13:32.154 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-07-28T19:13:32.170 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-07-28T19:13:32.204 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-07-28T19:13:32.282 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-07-28T19:13:32.298 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-07-28T19:13:32.332 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-07-28T19:13:32.967 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package radosgw. 2024-07-28T19:13:32.994 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:33.033 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-07-28T19:13:33.104 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package radosgw. 2024-07-28T19:13:33.132 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:33.162 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-07-28T19:13:33.944 INFO:teuthology.orchestra.run.smithi100.stdout:Selecting previously unselected package rbd-fuse. 2024-07-28T19:13:33.960 INFO:teuthology.orchestra.run.smithi100.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:34.002 INFO:teuthology.orchestra.run.smithi100.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-07-28T19:13:34.049 INFO:teuthology.orchestra.run.smithi138.stdout:Selecting previously unselected package rbd-fuse. 2024-07-28T19:13:34.065 INFO:teuthology.orchestra.run.smithi138.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-07-28T19:13:34.107 INFO:teuthology.orchestra.run.smithi138.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-07-28T19:13:34.315 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-07-28T19:13:34.403 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-07-28T19:13:34.557 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-07-28T19:13:34.646 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-07-28T19:13:34.683 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up nvme-cli (1.16-3ubuntu0.2) ... 2024-07-28T19:13:34.747 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up nvme-cli (1.16-3ubuntu0.2) ... 2024-07-28T19:13:34.944 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-07-28T19:13:35.169 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-07-28T19:13:35.306 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-07-28T19:13:35.519 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-07-28T19:13:35.738 INFO:teuthology.orchestra.run.smithi100.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-07-28T19:13:35.942 INFO:teuthology.orchestra.run.smithi138.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-07-28T19:13:36.008 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-07-28T19:13:36.061 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-07-28T19:13:36.545 INFO:teuthology.orchestra.run.smithi138.stdout:Adding system user cephadm....done 2024-07-28T19:13:36.563 INFO:teuthology.orchestra.run.smithi100.stdout:Adding system user cephadm....done 2024-07-28T19:13:36.675 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-07-28T19:13:36.726 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-07-28T19:13:36.969 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-07-28T19:13:37.027 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-07-28T19:13:37.221 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-07-28T19:13:37.253 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-07-28T19:13:37.447 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-07-28T19:13:37.488 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-07-28T19:13:37.707 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-07-28T19:13:37.748 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-07-28T19:13:37.833 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-py (1.10.0-1) ... 2024-07-28T19:13:37.874 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-py (1.10.0-1) ... 2024-07-28T19:13:38.135 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-07-28T19:13:38.175 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-07-28T19:13:38.495 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-07-28T19:13:38.544 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-07-28T19:13:38.730 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-07-28T19:13:38.778 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-07-28T19:13:38.998 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-07-28T19:13:39.038 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-07-28T19:13:39.241 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-07-28T19:13:39.289 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-07-28T19:13:39.500 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-07-28T19:13:39.556 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-07-28T19:13:39.760 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-07-28T19:13:39.833 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-07-28T19:13:40.028 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-07-28T19:13:40.108 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-07-28T19:13:40.145 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-07-28T19:13:40.235 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-07-28T19:13:40.380 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-07-28T19:13:40.490 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-07-28T19:13:40.502 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-07-28T19:13:40.603 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-07-28T19:13:40.849 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-07-28T19:13:40.946 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-07-28T19:13:40.975 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-07-28T19:13:41.072 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-07-28T19:13:41.243 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-07-28T19:13:41.340 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-07-28T19:13:41.511 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-07-28T19:13:41.616 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-07-28T19:13:41.813 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-07-28T19:13:41.917 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-07-28T19:13:42.465 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-07-28T19:13:42.553 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-07-28T19:13:42.700 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-07-28T19:13:42.812 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-07-28T19:13:42.826 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-07-28T19:13:42.921 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-07-28T19:13:42.952 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-07-28T19:13:43.048 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-07-28T19:13:43.262 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-07-28T19:13:43.349 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-07-28T19:13:43.505 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-07-28T19:13:43.633 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-07-28T19:13:43.865 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-07-28T19:13:44.018 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-07-28T19:13:44.150 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-07-28T19:13:44.319 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-07-28T19:13:44.518 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-07-28T19:13:44.637 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-07-28T19:13:44.770 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-07-28T19:13:44.863 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-07-28T19:13:44.887 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-07-28T19:13:45.023 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-07-28T19:13:45.181 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-07-28T19:13:45.316 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-07-28T19:13:45.499 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-07-28T19:13:45.642 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-07-28T19:13:46.689 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:13:46.823 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-toml (0.10.2-1) ... 2024-07-28T19:13:46.848 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:13:46.982 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-toml (0.10.2-1) ... 2024-07-28T19:13:47.075 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-07-28T19:13:47.201 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-07-28T19:13:47.241 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-07-28T19:13:47.327 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-07-28T19:13:47.367 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-07-28T19:13:47.493 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-07-28T19:13:47.637 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-07-28T19:13:47.795 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-07-28T19:13:47.872 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-07-28T19:13:48.046 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-07-28T19:13:48.123 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:13:48.249 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-07-28T19:13:48.297 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:13:48.376 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-07-28T19:13:48.423 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-07-28T19:13:48.549 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-07-28T19:13:48.677 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-07-28T19:13:48.851 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-07-28T19:13:48.928 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-07-28T19:13:49.118 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-07-28T19:13:49.171 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-07-28T19:13:49.298 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-07-28T19:13:49.386 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-07-28T19:13:49.487 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-07-28T19:13:49.549 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-07-28T19:13:49.738 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-07-28T19:13:49.802 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-07-28T19:13:49.989 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-07-28T19:13:50.077 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-07-28T19:13:50.203 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-07-28T19:13:50.282 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-07-28T19:13:50.408 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-07-28T19:13:50.479 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-07-28T19:13:50.589 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-07-28T19:13:50.709 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-07-28T19:13:50.836 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-07-28T19:13:50.874 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-07-28T19:13:51.128 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-07-28T19:13:51.167 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-07-28T19:13:51.414 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-07-28T19:13:51.441 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-07-28T19:13:51.569 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-07-28T19:13:51.663 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-07-28T19:13:51.790 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-07-28T19:13:51.983 INFO:teuthology.orchestra.run.smithi138.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-07-28T19:13:52.158 INFO:teuthology.orchestra.run.smithi100.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-07-28T19:13:52.231 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-07-28T19:13:52.357 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-07-28T19:13:52.409 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-07-28T19:13:52.483 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-07-28T19:13:52.535 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-07-28T19:13:52.661 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-07-28T19:13:52.893 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-07-28T19:13:53.054 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-07-28T19:13:53.153 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-portend (3.0.0-1) ... 2024-07-28T19:13:53.314 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-portend (3.0.0-1) ... 2024-07-28T19:13:53.404 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:13:53.539 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-07-28T19:13:53.590 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-07-28T19:13:53.657 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-07-28T19:13:53.716 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-07-28T19:13:53.842 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-07-28T19:13:53.958 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-07-28T19:13:54.084 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-07-28T19:13:54.160 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-07-28T19:13:54.286 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-07-28T19:13:54.386 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-07-28T19:13:54.570 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-07-28T19:13:54.670 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-07-28T19:13:54.796 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-07-28T19:13:54.813 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-07-28T19:13:54.931 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-rsa (4.8-1) ... 2024-07-28T19:13:54.939 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-07-28T19:13:55.065 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-rsa (4.8-1) ... 2024-07-28T19:13:55.182 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-07-28T19:13:55.341 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-07-28T19:13:55.442 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-07-28T19:13:55.600 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-07-28T19:13:55.669 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-07-28T19:13:55.669 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-07-28T19:13:55.814 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-07-28T19:13:55.814 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-07-28T19:13:56.144 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-07-28T19:13:56.303 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-07-28T19:13:56.336 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-07-28T19:13:56.436 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-07-28T19:13:56.563 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-07-28T19:13:56.687 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-07-28T19:13:57.089 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-07-28T19:13:57.224 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-07-28T19:13:57.373 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-07-28T19:13:57.534 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-07-28T19:13:57.795 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-07-28T19:13:57.896 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-07-28T19:13:57.948 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-07-28T19:13:58.022 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-07-28T19:13:58.074 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-07-28T19:13:58.148 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-07-28T19:13:58.201 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-07-28T19:13:58.274 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-07-28T19:13:58.327 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-07-28T19:13:58.400 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-07-28T19:13:58.453 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-07-28T19:13:58.571 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-07-28T19:13:58.702 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-07-28T19:13:58.811 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-07-28T19:13:58.840 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-07-28T19:13:58.921 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-07-28T19:13:58.966 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-07-28T19:13:59.092 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-07-28T19:14:00.110 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-07-28T19:14:00.239 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-07-28T19:14:00.587 INFO:teuthology.orchestra.run.smithi100.stdout:Adding group ceph....done 2024-07-28T19:14:00.817 INFO:teuthology.orchestra.run.smithi138.stdout:Adding group ceph....done 2024-07-28T19:14:00.971 INFO:teuthology.orchestra.run.smithi100.stdout:Adding system user ceph....done 2024-07-28T19:14:01.182 INFO:teuthology.orchestra.run.smithi100.stdout:Setting system user ceph properties....done 2024-07-28T19:14:01.193 INFO:teuthology.orchestra.run.smithi138.stdout:Adding system user ceph....done 2024-07-28T19:14:01.200 INFO:teuthology.orchestra.run.smithi100.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-07-28T19:14:01.335 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-07-28T19:14:01.345 INFO:teuthology.orchestra.run.smithi138.stdout:Setting system user ceph properties....done 2024-07-28T19:14:01.363 INFO:teuthology.orchestra.run.smithi138.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-07-28T19:14:01.492 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-07-28T19:14:01.680 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-07-28T19:14:01.831 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-07-28T19:14:02.186 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-07-28T19:14:02.295 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-07-28T19:14:02.309 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-07-28T19:14:02.418 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-07-28T19:14:02.543 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-07-28T19:14:02.543 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-07-28T19:14:02.675 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-07-28T19:14:02.675 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-07-28T19:14:03.197 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-07-28T19:14:03.337 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-07-28T19:14:03.546 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-07-28T19:14:03.661 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-07-28T19:14:04.024 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-07-28T19:14:04.106 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-07-28T19:14:04.252 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-07-28T19:14:04.252 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-07-28T19:14:04.340 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-07-28T19:14:04.340 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-07-28T19:14:04.743 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-07-28T19:14:04.800 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-07-28T19:14:04.939 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-07-28T19:14:04.939 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-07-28T19:14:05.019 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-07-28T19:14:05.019 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-07-28T19:14:05.377 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-07-28T19:14:05.485 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-07-28T19:14:05.647 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-07-28T19:14:05.647 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-07-28T19:14:05.773 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-07-28T19:14:05.773 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-07-28T19:14:06.104 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-07-28T19:14:06.247 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-07-28T19:14:06.270 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-07-28T19:14:06.371 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-07-28T19:14:06.389 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-07-28T19:14:06.514 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-07-28T19:14:06.619 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-07-28T19:14:06.619 INFO:teuthology.orchestra.run.smithi100.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-07-28T19:14:06.726 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-07-28T19:14:06.726 INFO:teuthology.orchestra.run.smithi138.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-07-28T19:14:07.099 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-07-28T19:14:07.208 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-07-28T19:14:07.250 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-07-28T19:14:07.342 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-07-28T19:14:07.376 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-07-28T19:14:07.469 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-07-28T19:14:07.510 INFO:teuthology.orchestra.run.smithi100.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-07-28T19:14:07.613 INFO:teuthology.orchestra.run.smithi100.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-07-28T19:14:07.620 INFO:teuthology.orchestra.run.smithi138.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-07-28T19:14:07.730 INFO:teuthology.orchestra.run.smithi138.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-07-28T19:14:08.756 INFO:teuthology.orchestra.run.smithi100.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-07-28T19:14:08.783 INFO:teuthology.orchestra.run.smithi138.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-07-28T19:14:09.243 INFO:teuthology.orchestra.run.smithi100.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-07-28T19:14:09.248 DEBUG:teuthology.parallel:result is None 2024-07-28T19:14:09.260 INFO:teuthology.orchestra.run.smithi138.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-07-28T19:14:09.264 DEBUG:teuthology.parallel:result is None 2024-07-28T19:14:09.265 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-28T19:14:09.265 INFO:teuthology.packaging:ref: None 2024-07-28T19:14:09.265 INFO:teuthology.packaging:tag: None 2024-07-28T19:14:09.265 INFO:teuthology.packaging:branch: quincy 2024-07-28T19:14:09.265 INFO:teuthology.packaging:sha1: 1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:14:09.266 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-07-28T19:14:09.406 DEBUG:teuthology.orchestra.run.smithi100:> dpkg-query -W -f '${Version}' ceph 2024-07-28T19:14:09.424 INFO:teuthology.orchestra.run.smithi100.stdout:17.2.7-1jammy 2024-07-28T19:14:09.425 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-07-28T19:14:09.425 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-07-28T19:14:09.427 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-07-28T19:14:09.427 INFO:teuthology.packaging:ref: None 2024-07-28T19:14:09.427 INFO:teuthology.packaging:tag: None 2024-07-28T19:14:09.427 INFO:teuthology.packaging:branch: quincy 2024-07-28T19:14:09.427 INFO:teuthology.packaging:sha1: 1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:14:09.427 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-07-28T19:14:09.572 DEBUG:teuthology.orchestra.run.smithi138:> dpkg-query -W -f '${Version}' ceph 2024-07-28T19:14:09.590 INFO:teuthology.orchestra.run.smithi138.stdout:17.2.7-1jammy 2024-07-28T19:14:09.591 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-07-28T19:14:09.591 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-07-28T19:14:09.593 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-07-28T19:14:09.593 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:14:09.593 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-07-28T19:14:09.607 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:14:09.607 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-07-28T19:14:09.646 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-07-28T19:14:09.646 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:14:09.646 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/usr/bin/daemon-helper 2024-07-28T19:14:09.664 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-07-28T19:14:09.719 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:14:09.720 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/daemon-helper 2024-07-28T19:14:09.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-07-28T19:14:09.786 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-07-28T19:14:09.786 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:14:09.787 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/usr/bin/adjust-ulimits 2024-07-28T19:14:09.799 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-07-28T19:14:09.851 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:14:09.851 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/adjust-ulimits 2024-07-28T19:14:09.863 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-07-28T19:14:09.918 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-07-28T19:14:09.919 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:14:09.919 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/usr/bin/stdin-killer 2024-07-28T19:14:09.932 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-07-28T19:14:09.987 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:14:09.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/stdin-killer 2024-07-28T19:14:09.999 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-07-28T19:14:10.054 INFO:teuthology.run_tasks:Running task cephadm... 2024-07-28T19:14:10.149 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64'} 2024-07-28T19:14:10.150 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-07-28T19:14:10.150 INFO:tasks.cephadm:Cluster fsid is 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:14:10.150 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-07-28T19:14:10.150 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.100', 'mon.c': '[v2:172.21.15.100:3301,v1:172.21.15.100:6790]', 'mon.b': '172.21.15.138'} 2024-07-28T19:14:10.151 INFO:tasks.cephadm:First mon is mon.a on smithi100 2024-07-28T19:14:10.151 INFO:tasks.cephadm:First mgr is y 2024-07-28T19:14:10.151 INFO:tasks.cephadm:Normalizing hostnames... 2024-07-28T19:14:10.151 DEBUG:teuthology.orchestra.run.smithi100:> sudo hostname $(hostname -s) 2024-07-28T19:14:10.165 DEBUG:teuthology.orchestra.run.smithi138:> sudo hostname $(hostname -s) 2024-07-28T19:14:10.179 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-07-28T19:14:10.179 DEBUG:teuthology.orchestra.run.smithi100:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-07-28T19:14:10.443 INFO:teuthology.orchestra.run.smithi100.stdout:-rw-rw-r-- 1 ubuntu ubuntu 375514 Jul 28 19:14 /home/ubuntu/cephtest/cephadm 2024-07-28T19:14:10.443 DEBUG:teuthology.orchestra.run.smithi138:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-07-28T19:14:10.664 INFO:teuthology.orchestra.run.smithi138.stdout:-rw-rw-r-- 1 ubuntu ubuntu 375514 Jul 28 19:14 /home/ubuntu/cephtest/cephadm 2024-07-28T19:14:10.665 DEBUG:teuthology.orchestra.run.smithi100:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-07-28T19:14:10.673 DEBUG:teuthology.orchestra.run.smithi138:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-07-28T19:14:10.693 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-07-28T19:14:10.693 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-07-28T19:14:10.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-07-28T19:14:10.883 INFO:teuthology.orchestra.run.smithi100.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-07-28T19:14:10.889 INFO:teuthology.orchestra.run.smithi138.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-07-28T19:15:43.916 INFO:teuthology.orchestra.run.smithi100.stdout:{ 2024-07-28T19:15:43.916 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph_version": "ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)", 2024-07-28T19:15:43.916 INFO:teuthology.orchestra.run.smithi100.stdout: "image_id": "5ec745195f7e964038144deb3b7cf6709e2a90e6b8f197967497f2a3bda375b9", 2024-07-28T19:15:43.916 INFO:teuthology.orchestra.run.smithi100.stdout: "repo_digests": [ 2024-07-28T19:15:43.916 INFO:teuthology.orchestra.run.smithi100.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:aef3335cf6bb52b9399a204b3a450d76e4542d974c0168c8be53fd7eddf1d5ca" 2024-07-28T19:15:43.916 INFO:teuthology.orchestra.run.smithi100.stdout: ] 2024-07-28T19:15:43.916 INFO:teuthology.orchestra.run.smithi100.stdout:} 2024-07-28T19:15:46.096 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2024-07-28T19:15:46.096 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph_version": "ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)", 2024-07-28T19:15:46.096 INFO:teuthology.orchestra.run.smithi138.stdout: "image_id": "5ec745195f7e964038144deb3b7cf6709e2a90e6b8f197967497f2a3bda375b9", 2024-07-28T19:15:46.096 INFO:teuthology.orchestra.run.smithi138.stdout: "repo_digests": [ 2024-07-28T19:15:46.097 INFO:teuthology.orchestra.run.smithi138.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:aef3335cf6bb52b9399a204b3a450d76e4542d974c0168c8be53fd7eddf1d5ca" 2024-07-28T19:15:46.097 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2024-07-28T19:15:46.097 INFO:teuthology.orchestra.run.smithi138.stdout:} 2024-07-28T19:15:46.123 DEBUG:teuthology.orchestra.run.smithi100:> sudo mkdir -p /etc/ceph 2024-07-28T19:15:46.138 DEBUG:teuthology.orchestra.run.smithi138:> sudo mkdir -p /etc/ceph 2024-07-28T19:15:46.151 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod 777 /etc/ceph 2024-07-28T19:15:46.194 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 777 /etc/ceph 2024-07-28T19:15:46.206 INFO:tasks.cephadm:Writing seed config... 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-07-28T19:15:46.208 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-07-28T19:15:46.209 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:15:46.210 DEBUG:teuthology.orchestra.run.smithi100:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-07-28T19:15:46.242 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9170a172-4d15-11ef-bcc8-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-07-28T19:15:46.243 DEBUG:teuthology.orchestra.run.smithi100:mon.a> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a.service 2024-07-28T19:15:46.287 DEBUG:teuthology.orchestra.run.smithi100:mgr.y> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y.service 2024-07-28T19:15:46.331 INFO:tasks.cephadm:Bootstrapping... 2024-07-28T19:15:46.331 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.100 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-07-28T19:15:46.542 INFO:teuthology.orchestra.run.smithi100.stdout:-------------------------------------------------------------------------------- 2024-07-28T19:15:46.542 INFO:teuthology.orchestra.run.smithi100.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '9170a172-4d15-11ef-bcc8-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.100', '--skip-admin-label'] 2024-07-28T19:15:46.542 INFO:teuthology.orchestra.run.smithi100.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-07-28T19:15:46.543 INFO:teuthology.orchestra.run.smithi100.stdout:Verifying podman|docker is present... 2024-07-28T19:15:46.543 INFO:teuthology.orchestra.run.smithi100.stdout:Verifying lvm2 is present... 2024-07-28T19:15:46.543 INFO:teuthology.orchestra.run.smithi100.stdout:Verifying time synchronization is in place... 2024-07-28T19:15:46.548 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-07-28T19:15:46.548 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-07-28T19:15:46.551 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-07-28T19:15:46.551 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout inactive 2024-07-28T19:15:46.555 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-07-28T19:15:46.555 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-07-28T19:15:46.558 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-07-28T19:15:46.559 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout inactive 2024-07-28T19:15:46.562 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-07-28T19:15:46.562 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout masked 2024-07-28T19:15:46.565 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-07-28T19:15:46.565 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout inactive 2024-07-28T19:15:46.569 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-07-28T19:15:46.569 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-07-28T19:15:46.574 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-07-28T19:15:46.574 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout inactive 2024-07-28T19:15:46.579 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout enabled 2024-07-28T19:15:46.583 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout active 2024-07-28T19:15:46.583 INFO:teuthology.orchestra.run.smithi100.stdout:Unit ntp.service is enabled and running 2024-07-28T19:15:46.583 INFO:teuthology.orchestra.run.smithi100.stdout:Repeating the final host check... 2024-07-28T19:15:46.583 INFO:teuthology.orchestra.run.smithi100.stdout:docker (/usr/bin/docker) is present 2024-07-28T19:15:46.583 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl is present 2024-07-28T19:15:46.583 INFO:teuthology.orchestra.run.smithi100.stdout:lvcreate is present 2024-07-28T19:15:46.586 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-07-28T19:15:46.586 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-07-28T19:15:46.589 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-07-28T19:15:46.589 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout inactive 2024-07-28T19:15:46.593 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-07-28T19:15:46.593 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-07-28T19:15:46.598 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-07-28T19:15:46.598 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout inactive 2024-07-28T19:15:46.601 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-07-28T19:15:46.601 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout masked 2024-07-28T19:15:46.606 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-07-28T19:15:46.606 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout inactive 2024-07-28T19:15:46.609 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-07-28T19:15:46.609 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-07-28T19:15:46.613 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-07-28T19:15:46.613 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout inactive 2024-07-28T19:15:46.617 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout enabled 2024-07-28T19:15:46.621 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stdout active 2024-07-28T19:15:46.621 INFO:teuthology.orchestra.run.smithi100.stdout:Unit ntp.service is enabled and running 2024-07-28T19:15:46.621 INFO:teuthology.orchestra.run.smithi100.stdout:Host looks OK 2024-07-28T19:15:46.621 INFO:teuthology.orchestra.run.smithi100.stdout:Cluster fsid: 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:15:46.622 INFO:teuthology.orchestra.run.smithi100.stdout:Acquiring lock 139922891614496 on /run/cephadm/9170a172-4d15-11ef-bcc8-c7b262605968.lock 2024-07-28T19:15:46.622 INFO:teuthology.orchestra.run.smithi100.stdout:Lock 139922891614496 acquired on /run/cephadm/9170a172-4d15-11ef-bcc8-c7b262605968.lock 2024-07-28T19:15:46.622 INFO:teuthology.orchestra.run.smithi100.stdout:Verifying IP 172.21.15.100 port 3300 ... 2024-07-28T19:15:46.622 INFO:teuthology.orchestra.run.smithi100.stdout:Verifying IP 172.21.15.100 port 6789 ... 2024-07-28T19:15:46.622 INFO:teuthology.orchestra.run.smithi100.stdout:Base mon IP(s) is [172.21.15.100:3300, 172.21.15.100:6789], mon addrv is [v2:172.21.15.100:3300,v1:172.21.15.100:6789] 2024-07-28T19:15:46.624 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2024-07-28T19:15:46.624 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.100 metric 100 2024-07-28T19:15:46.624 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout 158.69.67.47 via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.100 metric 100 2024-07-28T19:15:46.625 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2024-07-28T19:15:46.625 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.100 metric 100 2024-07-28T19:15:46.625 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout 172.21.0.1 dev enp3s0f1 proto dhcp scope link src 172.21.15.100 metric 100 2024-07-28T19:15:46.625 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout 172.21.0.2 dev enp3s0f1 proto dhcp scope link src 172.21.15.100 metric 100 2024-07-28T19:15:46.625 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.100 metric 100 2024-07-28T19:15:46.626 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-07-28T19:15:46.626 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-07-28T19:15:46.626 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1776sec pref medium 2024-07-28T19:15:46.628 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-07-28T19:15:46.628 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2024-07-28T19:15:46.628 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-07-28T19:15:46.628 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-07-28T19:15:46.629 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d733/64 scope link 2024-07-28T19:15:46.629 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Mon IP `172.21.15.100` is in CIDR network `172.21.0.0/20` 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Mon IP `172.21.15.100` is in CIDR network `172.21.0.0/20` 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Mon IP `172.21.15.100` is in CIDR network `172.21.0.1/32` 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Mon IP `172.21.15.100` is in CIDR network `172.21.0.1/32` 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Mon IP `172.21.15.100` is in CIDR network `172.21.0.2/32` 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Mon IP `172.21.15.100` is in CIDR network `172.21.0.2/32` 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Mon IP `172.21.15.100` is in CIDR network `172.21.15.254/32` 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Mon IP `172.21.15.100` is in CIDR network `172.21.15.254/32` 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.0.1/32', '172.21.0.1/32', '172.21.0.2/32', '172.21.0.2/32', '172.21.15.254/32', '172.21.15.254/32'] 2024-07-28T19:15:46.630 INFO:teuthology.orchestra.run.smithi100.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-07-28T19:15:46.631 INFO:teuthology.orchestra.run.smithi100.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-07-28T19:15:47.021 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/docker: stdout quincy: Pulling from ceph-ci/ceph 2024-07-28T19:15:47.021 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/docker: stdout Digest: sha256:aef3335cf6bb52b9399a204b3a450d76e4542d974c0168c8be53fd7eddf1d5ca 2024-07-28T19:15:47.022 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:quincy 2024-07-28T19:15:47.022 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:quincy 2024-07-28T19:15:48.710 INFO:teuthology.orchestra.run.smithi100.stdout:ceph: stdout ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable) 2024-07-28T19:15:48.710 INFO:teuthology.orchestra.run.smithi100.stdout:Ceph version: ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable) 2024-07-28T19:15:48.710 INFO:teuthology.orchestra.run.smithi100.stdout:Extracting ceph user uid/gid from container image... 2024-07-28T19:15:50.317 INFO:teuthology.orchestra.run.smithi100.stdout:stat: stdout 167 167 2024-07-28T19:15:50.317 INFO:teuthology.orchestra.run.smithi100.stdout:Creating initial keys... 2024-07-28T19:15:51.939 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-authtool: stdout AQDnmKZmLZKXFBAATXDFTEOWRmuRfIlPrmPLaA== 2024-07-28T19:15:53.520 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-authtool: stdout AQDomKZmgUGNNxAAHWKPxk4b/DqhZHnL1MDA7Q== 2024-07-28T19:15:55.134 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-authtool: stdout AQDqmKZmY9npIBAAn6obHdGeaLhCXgUhjBA0oQ== 2024-07-28T19:15:55.135 INFO:teuthology.orchestra.run.smithi100.stdout:Creating initial monmap... 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout:monmaptool for a [v2:172.21.15.100:3300,v1:172.21.15.100:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout:setting min_mon_release = octopus 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/monmaptool: set fsid to 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-07-28T19:15:56.658 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:15:56.659 INFO:teuthology.orchestra.run.smithi100.stdout:Creating mon... 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.648+0000 7f49dfe54900 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.648+0000 7f49dfe54900 1 imported monmap: 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr epoch 0 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr last_changed 2024-07-28T19:15:56.118341+0000 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr created 2024-07-28T19:15:56.118341+0000 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr min_mon_release 15 (octopus) 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon.a 2024-07-28T19:15:58.314 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.648+0000 7f49dfe54900 0 /usr/bin/ceph-mon: set fsid to 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: RocksDB version: 6.15.5 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Compile date Jul 28 2024 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: DB SUMMARY 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: DB Session ID: 23W3NVNN311HINQPF61Q 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.315 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.error_if_exists: 0 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.create_if_missing: 1 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.paranoid_checks: 1 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.env: 0x55c968423440 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.fs: Posix File System 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.info_log: 0x55c969be66a0 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.statistics: (nil) 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.use_fsync: 0 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_log_file_size: 0 2024-07-28T19:15:58.316 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.allow_fallocate: 1 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.use_direct_reads: 0 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.create_missing_column_families: 0 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.db_log_dir: 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-28T19:15:58.317 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.advise_random_on_open: 1 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.write_buffer_manager: 0x55c969c2fe00 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.rate_limiter: (nil) 2024-07-28T19:15:58.318 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.unordered_write: 0 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.row_cache: None 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.wal_filter: None 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-28T19:15:58.319 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.preserve_deletes: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.two_write_queues: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.manual_wal_flush: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.atomic_flush: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.log_readahead_size: 0 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-28T19:15:58.320 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_background_jobs: 2 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_background_compactions: -1 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_subcompactions: 1 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_total_wal_size: 0 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-28T19:15:58.321 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_open_files: -1 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bytes_per_sync: 0 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_background_flushes: -1 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Compression algorithms supported: 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: kZSTD supported: 0 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: kXpressCompression supported: 0 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: kBZip2Compression supported: 0 2024-07-28T19:15:58.322 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: kLZ4Compression supported: 1 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: kZlibCompression supported: 1 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: kSnappyCompression supported: 1 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: [db/db_impl/db_impl_open.cc:281] Creating manifest 1 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-28T19:15:58.323 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.merge_operator: 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_filter: None 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_filter_factory: None 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c969b87ef8) 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2024-07-28T19:15:58.324 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr hash_index_allow_collision: 1 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr checksum: 1 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x55c969bbf1f0 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2024-07-28T19:15:58.325 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr filter_policy: rocksdb.BuiltinBloomFilter 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr format_version: 4 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.326 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compression: NoCompression 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.num_levels: 7 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-28T19:15:58.327 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compression_opts.level: 32767 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compression_opts.enabled: false 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-28T19:15:58.328 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-28T19:15:58.329 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.arena_block_size: 4194304 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-28T19:15:58.330 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.table_properties_collectors: 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.inplace_update_support: 0 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.bloom_locality: 0 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.max_successive_merges: 0 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-28T19:15:58.331 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.force_consistency_checks: 1 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.ttl: 2592000 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.enable_blob_files: false 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.min_blob_size: 0 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.blob_file_size: 268435456 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-07-28T19:15:58.332 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 3 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55c969bf0700 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.652+0000 7f49dfe54900 4 rocksdb: DB pointer 0x55c969bc2000 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.656+0000 7f49d75e2640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.656+0000 7f49d75e2640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-07-28T19:15:58.333 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-07-28T19:15:58.334 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:15:58.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:15:58.336 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:15:58.337 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:15:58.337 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.337 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:15:58.337 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr 2024-07-28T19:15:58.337 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.656+0000 7f49dfe54900 4 rocksdb: [db/db_impl/db_impl.cc:446] Shutdown: canceling all background work 2024-07-28T19:15:58.337 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.656+0000 7f49dfe54900 4 rocksdb: [db/db_impl/db_impl.cc:625] Shutdown complete 2024-07-28T19:15:58.337 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph-mon: stderr debug 2024-07-28T19:15:57.656+0000 7f49dfe54900 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-07-28T19:15:58.337 INFO:teuthology.orchestra.run.smithi100.stdout:create mon.a on 2024-07-28T19:15:58.539 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-07-28T19:15:58.726 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2024-07-28T19:15:58.921 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9170a172-4d15-11ef-bcc8-c7b262605968.target -> /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968.target. 2024-07-28T19:15:58.921 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-9170a172-4d15-11ef-bcc8-c7b262605968.target -> /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968.target. 2024-07-28T19:15:59.121 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a 2024-07-28T19:15:59.122 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Failed to reset failed state of unit ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a.service: Unit ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a.service not loaded. 2024-07-28T19:15:59.306 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968.target.wants/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a.service -> /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service. 2024-07-28T19:15:59.315 INFO:teuthology.orchestra.run.smithi100.stdout:firewalld does not appear to be present 2024-07-28T19:15:59.315 INFO:teuthology.orchestra.run.smithi100.stdout:Not possible to enable service . firewalld.service is not available 2024-07-28T19:15:59.316 INFO:teuthology.orchestra.run.smithi100.stdout:Waiting for mon to start... 2024-07-28T19:15:59.316 INFO:teuthology.orchestra.run.smithi100.stdout:Waiting for mon... 2024-07-28T19:15:59.441 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:15:59 smithi100 systemd[1]: Started Ceph mon.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:16:00.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:00 smithi100 bash[21350]: cluster 2024-07-28T19:16:00.842518+0000 mon.a (mon.0) 0 : cluster [INF] mkfs 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:16:00.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:00 smithi100 bash[21350]: cluster 2024-07-28T19:16:00.817536+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-07-28T19:16:01.595 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout cluster: 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout id: 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout services: 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.162946s) 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout data: 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout pgs: 2024-07-28T19:16:01.596 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:01.597 INFO:teuthology.orchestra.run.smithi100.stdout:mon is available 2024-07-28T19:16:01.597 INFO:teuthology.orchestra.run.smithi100.stdout:Assimilating anything we can from ceph.conf... 2024-07-28T19:16:01.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:01 smithi100 bash[21350]: cluster 2024-07-28T19:16:00.841471+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-07-28T19:16:01.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:01 smithi100 bash[21350]: cluster 2024-07-28T19:16:00.841701+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:16:01.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:01 smithi100 bash[21350]: cluster 2024-07-28T19:16:00.844185+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2024-07-28T19:16:01.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:01 smithi100 bash[21350]: cluster 2024-07-28T19:16:00.846491+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-07-28T19:16:01.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:01 smithi100 bash[21350]: cluster 2024-07-28T19:16:00.846908+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2024-07-28T19:16:01.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:01 smithi100 bash[21350]: audit 2024-07-28T19:16:01.004368+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.100:0/852825100' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-07-28T19:16:03.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:02 smithi100 bash[21350]: audit 2024-07-28T19:16:02.865191+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.100:0/1837012175' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-07-28T19:16:03.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:02 smithi100 bash[21350]: audit 2024-07-28T19:16:02.870773+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.100:0/1837012175' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-07-28T19:16:03.484 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout [global] 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout fsid = 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.100:3300,v1:172.21.15.100:6789] 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout [mgr] 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout [osd] 2024-07-28T19:16:03.485 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-07-28T19:16:03.486 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-07-28T19:16:03.486 INFO:teuthology.orchestra.run.smithi100.stdout:Generating new minimal ceph.conf... 2024-07-28T19:16:04.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:04 smithi100 bash[21350]: audit 2024-07-28T19:16:04.653230+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.100:0/451155736' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:16:05.157 INFO:teuthology.orchestra.run.smithi100.stdout:Restarting the monitor... 2024-07-28T19:16:05.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:05 smithi100 systemd[1]: Stopping Ceph mon.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:16:05.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:05 smithi100 bash[21350]: debug 2024-07-28T19:16:05.275+0000 7fa5ad5dc640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-28T19:16:05.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:05 smithi100 bash[21350]: debug 2024-07-28T19:16:05.275+0000 7fa5ad5dc640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-07-28T19:16:06.065 INFO:teuthology.orchestra.run.smithi100.stdout:Setting public_network to 172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20 in mon config section 2024-07-28T19:16:06.191 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:05 smithi100 bash[21730]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-a 2024-07-28T19:16:06.191 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:06 smithi100 bash[21790]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-a 2024-07-28T19:16:06.192 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:06 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a.service: Deactivated successfully. 2024-07-28T19:16:06.192 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:06 smithi100 systemd[1]: Stopped Ceph mon.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:16:06.192 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:06 smithi100 systemd[1]: Started Ceph mon.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:16:07.879 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.631+0000 7f9b00f81900 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-28T19:16:07.879 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.631+0000 7f9b00f81900 0 ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable), process ceph-mon, pid 7 2024-07-28T19:16:07.879 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.631+0000 7f9b00f81900 0 pidfile_write: ignore empty --pid-file 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 0 load: jerasure load: lrc 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: RocksDB version: 6.15.5 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Compile date Jul 28 2024 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: DB SUMMARY 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: DB Session ID: 5SJLUBXBJMQHKLZQK7O5 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: CURRENT file: CURRENT 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: IDENTITY file: IDENTITY 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83181 ; 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.error_if_exists: 0 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.create_if_missing: 0 2024-07-28T19:16:07.880 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.paranoid_checks: 1 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.env: 0x56050502b440 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.fs: Posix File System 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.info_log: 0x5605072e6f40 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.statistics: (nil) 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.use_fsync: 0 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_log_file_size: 0 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-28T19:16:07.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.allow_fallocate: 1 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.use_direct_reads: 0 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.create_missing_column_families: 0 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.db_log_dir: 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-28T19:16:07.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.advise_random_on_open: 1 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.write_buffer_manager: 0x5605073a46f0 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.rate_limiter: (nil) 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-28T19:16:07.883 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.unordered_write: 0 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.row_cache: None 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.wal_filter: None 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.preserve_deletes: 0 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.two_write_queues: 0 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.manual_wal_flush: 0 2024-07-28T19:16:07.884 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.atomic_flush: 0 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.log_readahead_size: 0 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_background_jobs: 2 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_background_compactions: -1 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_subcompactions: 1 2024-07-28T19:16:07.885 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-28T19:16:07.886 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_total_wal_size: 0 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_open_files: -1 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bytes_per_sync: 0 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-28T19:16:07.887 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_background_flushes: -1 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Compression algorithms supported: 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: kZSTD supported: 0 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: kXpressCompression supported: 0 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: kBZip2Compression supported: 0 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: kLZ4Compression supported: 1 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: kZlibCompression supported: 1 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: kSnappyCompression supported: 1 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-28T19:16:07.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.merge_operator: 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_filter: None 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_filter_factory: None 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56050728fa58) 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: cache_index_and_filter_blocks: 1 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: pin_top_level_index_and_filter: 1 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: index_type: 0 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: data_block_index_type: 0 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: index_shortening: 1 2024-07-28T19:16:07.889 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: data_block_hash_table_util_ratio: 0.750000 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: hash_index_allow_collision: 1 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: checksum: 1 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: no_block_cache: 0 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: block_cache: 0x5605072c31f0 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: block_cache_name: BinnedLRUCache 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: block_cache_options: 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: capacity : 536870912 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: num_shard_bits : 4 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: strict_capacity_limit : 0 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: high_pri_pool_ratio: 0.000 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: block_cache_compressed: (nil) 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: persistent_cache: (nil) 2024-07-28T19:16:07.890 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: block_size: 4096 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: block_size_deviation: 10 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: block_restart_interval: 16 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: index_block_restart_interval: 1 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: metadata_block_size: 4096 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: partition_filters: 0 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: use_delta_encoding: 1 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: filter_policy: rocksdb.BuiltinBloomFilter 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: whole_key_filtering: 1 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: verify_compression: 0 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: read_amp_bytes_per_bit: 0 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: format_version: 4 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: enable_index_compression: 1 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: block_align: 0 2024-07-28T19:16:07.891 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compression: NoCompression 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.num_levels: 7 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-28T19:16:07.892 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compression_opts.level: 32767 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compression_opts.enabled: false 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-28T19:16:07.893 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.arena_block_size: 4194304 2024-07-28T19:16:07.894 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-28T19:16:07.895 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.table_properties_collectors: 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.inplace_update_support: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.bloom_locality: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.max_successive_merges: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.force_consistency_checks: 1 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.ttl: 2592000 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.enable_blob_files: false 2024-07-28T19:16:07.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.min_blob_size: 0 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.blob_file_size: 268435456 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194167640612, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.635+0000 7f9b00f81900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9b00f81900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9b00f81900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194167641630, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79746, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78011, "index_size": 223, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9999, "raw_average_key_size": 47, "raw_value_size": 72331, "raw_average_value_size": 342, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1722194167, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "f5893a4e-1044-4197-a091-cfdc818067c7", "db_session_id": "5SJLUBXBJMQHKLZQK7O5"}} 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9b00f81900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9b00f81900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194167642146, "job": 1, "event": "recovery_finished"} 2024-07-28T19:16:07.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9b00f81900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9b00f81900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5605072f8700 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9b00f81900 4 rocksdb: DB pointer 0x5605072c6000 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9af6d4d640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9af6d4d640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ** DB Stats ** 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:16:07.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ** Compaction Stats [default] ** 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: L0 2/0 79.64 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 112.3 0.00 0.00 1 0.001 0 0 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Sum 2/0 79.64 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 112.3 0.00 0.00 1 0.001 0 0 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 112.3 0.00 0.00 1 0.001 0 0 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ** Compaction Stats [default] ** 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 112.3 0.00 0.00 1 0.001 0 0 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:16:07.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Cumulative compaction: 0.00 GB write, 18.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Interval compaction: 0.00 GB write, 18.32 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ** Compaction Stats [default] ** 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: L0 2/0 79.64 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 112.3 0.00 0.00 1 0.001 0 0 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Sum 2/0 79.64 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 112.3 0.00 0.00 1 0.001 0 0 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ** Compaction Stats [default] ** 2024-07-28T19:16:07.900 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 112.3 0.00 0.00 1 0.001 0 0 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Cumulative compaction: 0.00 GB write, 18.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:16:07.901 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.639+0000 7f9b00f81900 0 starting mon.a rank 0 at public addrs [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] at bind addrs [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.643+0000 7f9b00f81900 1 mon.a@-1(???) e1 preinit fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.643+0000 7f9b00f81900 0 mon.a@-1(???).mds e1 new map 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.643+0000 7f9b00f81900 0 mon.a@-1(???).mds e1 print_map 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: e1 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: legacy client fscid: -1 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: No filesystems configured 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.643+0000 7f9b00f81900 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.643+0000 7f9b00f81900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.643+0000 7f9b00f81900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:16:07.902 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.643+0000 7f9b00f81900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:16:07.903 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: debug 2024-07-28T19:16:07.643+0000 7f9b00f81900 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-07-28T19:16:07.903 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: cluster 2024-07-28T19:16:07.669521+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-07-28T19:16:07.903 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: cluster 2024-07-28T19:16:07.669607+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:16:07.903 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: cluster 2024-07-28T19:16:07.670043+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2024-07-28T19:16:07.903 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: cluster 2024-07-28T19:16:07.670068+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-07-28T19:16:07.903 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:07 smithi100 bash[21828]: cluster 2024-07-28T19:16:07.670487+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2024-07-28T19:16:08.452 INFO:teuthology.orchestra.run.smithi100.stdout:Wrote config to /etc/ceph/ceph.conf 2024-07-28T19:16:08.453 INFO:teuthology.orchestra.run.smithi100.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-07-28T19:16:08.453 INFO:teuthology.orchestra.run.smithi100.stdout:Creating mgr... 2024-07-28T19:16:08.453 INFO:teuthology.orchestra.run.smithi100.stdout:Verifying port 9283 ... 2024-07-28T19:16:08.716 INFO:teuthology.orchestra.run.smithi100.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y 2024-07-28T19:16:08.717 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Failed to reset failed state of unit ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y.service: Unit ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y.service not loaded. 2024-07-28T19:16:08.721 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:16:08.721 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:16:08.912 INFO:teuthology.orchestra.run.smithi100.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968.target.wants/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y.service -> /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service. 2024-07-28T19:16:08.919 INFO:teuthology.orchestra.run.smithi100.stdout:firewalld does not appear to be present 2024-07-28T19:16:08.920 INFO:teuthology.orchestra.run.smithi100.stdout:Not possible to enable service . firewalld.service is not available 2024-07-28T19:16:08.920 INFO:teuthology.orchestra.run.smithi100.stdout:firewalld does not appear to be present 2024-07-28T19:16:08.920 INFO:teuthology.orchestra.run.smithi100.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-07-28T19:16:08.920 INFO:teuthology.orchestra.run.smithi100.stdout:Waiting for mgr to start... 2024-07-28T19:16:08.920 INFO:teuthology.orchestra.run.smithi100.stdout:Waiting for mgr... 2024-07-28T19:16:08.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:16:08.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:08 smithi100 systemd[1]: Started Ceph mgr.y for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:16:08.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:16:08.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:08 smithi100 bash[21828]: audit 2024-07-28T19:16:07.869959+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.100:0/919969316' entity='client.admin' 2024-07-28T19:16:10.748 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:10 smithi100 bash[22098]: debug 2024-07-28T19:16:10.630+0000 7f37c21baec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:16:11.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:10 smithi100 bash[21828]: audit 2024-07-28T19:16:10.779427+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.100:0/2198962943' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-07-28T19:16:11.382 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:11.382 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout { 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "fsid": "9170a172-4d15-11ef-bcc8-c7b262605968", 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "health": { 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 0 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "a" 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum_age": 3, 2024-07-28T19:16:11.383 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "monmap": { 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-07-28T19:16:11.384 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-07-28T19:16:11.387 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:11.387 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:11.388 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "available": false, 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "modules": [ 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "iostat", 2024-07-28T19:16:11.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "nfs", 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "restful" 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "modified": "2024-07-28T19:16:00.842943+0000", 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout } 2024-07-28T19:16:11.390 INFO:teuthology.orchestra.run.smithi100.stdout:mgr not available, waiting (1/15)... 2024-07-28T19:16:12.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:11 smithi100 bash[22098]: debug 2024-07-28T19:16:11.846+0000 7f37c21baec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:16:12.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:11 smithi100 bash[22098]: debug 2024-07-28T19:16:11.914+0000 7f37c21baec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:16:12.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:12 smithi100 bash[22098]: debug 2024-07-28T19:16:12.066+0000 7f37c21baec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:16:12.979 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:12 smithi100 bash[22098]: debug 2024-07-28T19:16:12.686+0000 7f37c21baec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:16:12.979 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:12 smithi100 bash[22098]: debug 2024-07-28T19:16:12.754+0000 7f37c21baec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:16:12.979 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:12 smithi100 bash[22098]: debug 2024-07-28T19:16:12.822+0000 7f37c21baec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:16:13.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:12 smithi100 bash[22098]: debug 2024-07-28T19:16:12.974+0000 7f37c21baec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:16:13.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:13 smithi100 bash[22098]: debug 2024-07-28T19:16:13.050+0000 7f37c21baec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:16:13.249 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:13 smithi100 bash[22098]: debug 2024-07-28T19:16:13.114+0000 7f37c21baec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:16:13.674 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:13 smithi100 bash[22098]: debug 2024-07-28T19:16:13.338+0000 7f37c21baec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:16:13.674 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:13 smithi100 bash[22098]: debug 2024-07-28T19:16:13.406+0000 7f37c21baec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:16:13.674 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:13 smithi100 bash[22098]: debug 2024-07-28T19:16:13.574+0000 7f37c21baec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:16:13.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:13 smithi100 bash[22098]: debug 2024-07-28T19:16:13.646+0000 7f37c21baec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:16:13.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:13 smithi100 bash[22098]: debug 2024-07-28T19:16:13.878+0000 7f37c21baec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:16:14.498 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:14 smithi100 bash[22098]: debug 2024-07-28T19:16:14.038+0000 7f37c21baec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:16:14.499 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:14 smithi100 bash[22098]: debug 2024-07-28T19:16:14.114+0000 7f37c21baec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:16:14.499 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:14 smithi100 bash[22098]: debug 2024-07-28T19:16:14.266+0000 7f37c21baec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:16:15.104 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[22098]: debug 2024-07-28T19:16:15.030+0000 7f37c21baec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:16:15.105 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:14 smithi100 bash[21828]: audit 2024-07-28T19:16:14.786587+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.100:0/1537147164' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-07-28T19:16:15.372 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[22098]: debug 2024-07-28T19:16:15.102+0000 7f37c21baec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:16:15.372 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[22098]: debug 2024-07-28T19:16:15.298+0000 7f37c21baec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:16:15.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:15.389 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout { 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "fsid": "9170a172-4d15-11ef-bcc8-c7b262605968", 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "health": { 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 0 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "a" 2024-07-28T19:16:15.390 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum_age": 7, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "monmap": { 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-07-28T19:16:15.391 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-07-28T19:16:15.395 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "available": false, 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "modules": [ 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "iostat", 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "nfs", 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "restful" 2024-07-28T19:16:15.396 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "modified": "2024-07-28T19:16:00.842943+0000", 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout } 2024-07-28T19:16:15.397 INFO:teuthology.orchestra.run.smithi100.stdout:mgr not available, waiting (2/15)... 2024-07-28T19:16:15.684 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[22098]: debug 2024-07-28T19:16:15.370+0000 7f37c21baec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[22098]: debug 2024-07-28T19:16:15.682+0000 7f37c21baec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[22098]: debug 2024-07-28T19:16:15.758+0000 7f37c21baec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: cluster 2024-07-28T19:16:15.782574+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: cluster 2024-07-28T19:16:15.788840+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00648518s) 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.789822+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.789929+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.790020+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.790220+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:16:15.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.791110+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:16:16.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: cluster 2024-07-28T19:16:15.815324+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2024-07-28T19:16:16.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.825375+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:16:16.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.827905+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:16:16.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.830779+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' 2024-07-28T19:16:16.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.832696+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' 2024-07-28T19:16:16.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:15 smithi100 bash[21828]: audit 2024-07-28T19:16:15.834733+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.100:0/2738314877' entity='mgr.y' 2024-07-28T19:16:18.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:17 smithi100 bash[21828]: cluster 2024-07-28T19:16:16.797716+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.01536s) 2024-07-28T19:16:19.630 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout { 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "fsid": "9170a172-4d15-11ef-bcc8-c7b262605968", 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "health": { 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-07-28T19:16:19.631 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 0 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "a" 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "monmap": { 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-07-28T19:16:19.632 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-07-28T19:16:19.633 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-07-28T19:16:19.633 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-07-28T19:16:19.633 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-07-28T19:16:19.633 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-07-28T19:16:19.636 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "available": true, 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "modules": [ 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "iostat", 2024-07-28T19:16:19.637 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "nfs", 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "restful" 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ], 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "modified": "2024-07-28T19:16:00.842943+0000", 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "services": {} 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout }, 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout } 2024-07-28T19:16:19.638 INFO:teuthology.orchestra.run.smithi100.stdout:mgr is available 2024-07-28T19:16:19.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:19 smithi100 bash[21828]: cluster 2024-07-28T19:16:18.796857+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e4: y(active, since 3s) 2024-07-28T19:16:19.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:19 smithi100 bash[21828]: audit 2024-07-28T19:16:19.043291+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.100:0/1710753866' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-07-28T19:16:21.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:21 smithi100 bash[21828]: audit 2024-07-28T19:16:21.021271+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.100:0/348125532' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-07-28T19:16:21.545 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:21.545 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout [global] 2024-07-28T19:16:21.545 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout fsid = 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:16:21.545 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-07-28T19:16:21.545 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-07-28T19:16:21.545 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-07-28T19:16:21.545 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout [mgr] 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout [osd] 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-07-28T19:16:21.546 INFO:teuthology.orchestra.run.smithi100.stdout:Enabling cephadm module... 2024-07-28T19:16:23.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:23 smithi100 bash[21828]: audit 2024-07-28T19:16:22.964605+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.100:0/1789641905' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-07-28T19:16:23.499 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:23 smithi100 bash[22098]: ignoring --setuser ceph since I am not root 2024-07-28T19:16:23.499 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:23 smithi100 bash[22098]: ignoring --setgroup ceph since I am not root 2024-07-28T19:16:23.499 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:23 smithi100 bash[22098]: debug 2024-07-28T19:16:23.153+0000 7f74c3c14ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:16:24.333 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:24 smithi100 bash[21828]: audit 2024-07-28T19:16:23.019919+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.100:0/1789641905' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-07-28T19:16:24.333 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:24 smithi100 bash[21828]: cluster 2024-07-28T19:16:23.019968+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 7s) 2024-07-28T19:16:24.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:24 smithi100 bash[22098]: debug 2024-07-28T19:16:24.329+0000 7f74c3c14ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:16:24.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:24 smithi100 bash[22098]: debug 2024-07-28T19:16:24.389+0000 7f74c3c14ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:16:24.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:24 smithi100 bash[22098]: debug 2024-07-28T19:16:24.549+0000 7f74c3c14ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:16:25.325 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[21828]: audit 2024-07-28T19:16:25.029236+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.100:0/1546429379' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-07-28T19:16:25.325 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[22098]: debug 2024-07-28T19:16:25.185+0000 7f74c3c14ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:16:25.325 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[22098]: debug 2024-07-28T19:16:25.253+0000 7f74c3c14ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:16:25.574 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout { 2024-07-28T19:16:25.574 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-07-28T19:16:25.574 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "available": true, 2024-07-28T19:16:25.574 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-07-28T19:16:25.574 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-07-28T19:16:25.574 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout } 2024-07-28T19:16:25.575 INFO:teuthology.orchestra.run.smithi100.stdout:Waiting for the mgr to restart... 2024-07-28T19:16:25.575 INFO:teuthology.orchestra.run.smithi100.stdout:Waiting for mgr epoch 5... 2024-07-28T19:16:25.623 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[22098]: debug 2024-07-28T19:16:25.321+0000 7f74c3c14ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:16:25.623 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[22098]: debug 2024-07-28T19:16:25.473+0000 7f74c3c14ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:16:25.623 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[22098]: debug 2024-07-28T19:16:25.549+0000 7f74c3c14ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:16:25.623 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[22098]: debug 2024-07-28T19:16:25.617+0000 7f74c3c14ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:16:25.915 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[22098]: debug 2024-07-28T19:16:25.845+0000 7f74c3c14ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:16:26.249 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:25 smithi100 bash[22098]: debug 2024-07-28T19:16:25.909+0000 7f74c3c14ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:16:26.249 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:26 smithi100 bash[22098]: debug 2024-07-28T19:16:26.077+0000 7f74c3c14ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:16:26.249 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:26 smithi100 bash[22098]: debug 2024-07-28T19:16:26.153+0000 7f74c3c14ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:16:26.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:26 smithi100 bash[22098]: debug 2024-07-28T19:16:26.385+0000 7f74c3c14ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:16:26.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:26 smithi100 bash[22098]: debug 2024-07-28T19:16:26.545+0000 7f74c3c14ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:16:26.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:26 smithi100 bash[22098]: debug 2024-07-28T19:16:26.625+0000 7f74c3c14ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:16:27.249 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:26 smithi100 bash[22098]: debug 2024-07-28T19:16:26.785+0000 7f74c3c14ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:16:27.807 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:27 smithi100 bash[22098]: debug 2024-07-28T19:16:27.537+0000 7f74c3c14ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:16:27.807 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:27 smithi100 bash[22098]: debug 2024-07-28T19:16:27.609+0000 7f74c3c14ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:16:28.188 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:27 smithi100 bash[22098]: debug 2024-07-28T19:16:27.801+0000 7f74c3c14ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:16:28.188 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:27 smithi100 bash[22098]: debug 2024-07-28T19:16:27.873+0000 7f74c3c14ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:16:28.341 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[22098]: debug 2024-07-28T19:16:28.185+0000 7f74c3c14ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:16:28.341 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[22098]: debug 2024-07-28T19:16:28.261+0000 7f74c3c14ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:16:28.342 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: cluster 2024-07-28T19:16:28.286804+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2024-07-28T19:16:28.342 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: cluster 2024-07-28T19:16:28.287341+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2024-07-28T19:16:28.342 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: cluster 2024-07-28T19:16:28.294033+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2024-07-28T19:16:28.342 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: cluster 2024-07-28T19:16:28.294270+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00709226s) 2024-07-28T19:16:28.342 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: audit 2024-07-28T19:16:28.295606+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:16:28.342 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: audit 2024-07-28T19:16:28.296711+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:16:28.342 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: audit 2024-07-28T19:16:28.298036+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:16:28.342 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: audit 2024-07-28T19:16:28.298265+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:16:28.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: audit 2024-07-28T19:16:28.298396+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:16:28.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:28 smithi100 bash[21828]: cluster 2024-07-28T19:16:28.323085+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2024-07-28T19:16:29.619 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: cephadm 2024-07-28T19:16:28.333062+0000 mgr.y (mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2024-07-28T19:16:29.619 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: audit 2024-07-28T19:16:28.338431+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:29.619 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: audit 2024-07-28T19:16:28.348035+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:29.619 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: audit 2024-07-28T19:16:28.350915+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:16:29.619 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: audit 2024-07-28T19:16:28.352666+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:16:29.619 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: audit 2024-07-28T19:16:28.355795+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:16:29.619 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: audit 2024-07-28T19:16:28.364480+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:16:29.619 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: audit 2024-07-28T19:16:28.367343+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:16:29.620 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[21828]: cluster 2024-07-28T19:16:29.298546+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.01136s) 2024-07-28T19:16:29.906 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout { 2024-07-28T19:16:29.906 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-07-28T19:16:29.906 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "initialized": true 2024-07-28T19:16:29.906 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout } 2024-07-28T19:16:29.906 INFO:teuthology.orchestra.run.smithi100.stdout:mgr epoch 5 is available 2024-07-28T19:16:29.906 INFO:teuthology.orchestra.run.smithi100.stdout:Setting orchestrator backend to cephadm... 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: debug 2024-07-28T19:16:29.613+0000 7f7476002640 -1 log_channel(cephadm) log [ERR] : [28/Jul/2024:19:16:29] ENGINE Error in HTTPServer.serve 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: Traceback (most recent call last): 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: self._connections.run(self.expiration_interval) 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: self._run(expiration_interval) 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:16:29.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: s = self.context.wrap_socket( 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: return self.sslsocket_class._create( 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: self.do_handshake() 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: self._sslobj.do_handshake() 2024-07-28T19:16:29.999 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:29 smithi100 bash[22098]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:16:30.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: audit 2024-07-28T19:16:29.295309+0000 mgr.y (mgr.14118) 2 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-07-28T19:16:30.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: audit 2024-07-28T19:16:29.307619+0000 mgr.y (mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: cephadm 2024-07-28T19:16:29.497810+0000 mgr.y (mgr.14118) 4 : cephadm [INF] [28/Jul/2024:19:16:29] ENGINE Bus STARTING 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: cephadm 2024-07-28T19:16:29.617929+0000 mgr.y (mgr.14118) 5 : cephadm [INF] [28/Jul/2024:19:16:29] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: cephadm 2024-07-28T19:16:29.618038+0000 mgr.y (mgr.14118) 6 : cephadm [INF] [28/Jul/2024:19:16:29] ENGINE Bus STARTED 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: cephadm 2024-07-28T19:16:29.619837+0000 mgr.y (mgr.14118) 7 : cephadm [ERR] [28/Jul/2024:19:16:29] ENGINE Error in HTTPServer.serve 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: Traceback (most recent call last): 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: self._connections.run(self.expiration_interval) 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: self._run(expiration_interval) 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:16:31.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: s = self.context.wrap_socket( 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: return self.sslsocket_class._create( 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: self.do_handshake() 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: self._sslobj.do_handshake() 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: audit 2024-07-28T19:16:29.624188+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:31.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:30 smithi100 bash[21828]: audit 2024-07-28T19:16:29.625296+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:16:31.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:31 smithi100 bash[21828]: cluster 2024-07-28T19:16:30.632635+0000 mon.a (mon.0) 50 : cluster [DBG] mgrmap e8: y(active, since 2s) 2024-07-28T19:16:31.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:31 smithi100 bash[21828]: audit 2024-07-28T19:16:31.356899+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:32.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:31 smithi100 bash[21828]: audit 2024-07-28T19:16:31.369878+0000 mon.a (mon.0) 52 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:16:32.886 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:32 smithi100 bash[21828]: audit 2024-07-28T19:16:31.347811+0000 mgr.y (mgr.14118) 8 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:16:33.929 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout value unchanged 2024-07-28T19:16:33.929 INFO:teuthology.orchestra.run.smithi100.stdout:Generating ssh key... 2024-07-28T19:16:34.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:34 smithi100 bash[21828]: audit 2024-07-28T19:16:33.371408+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: Generating public/private rsa key pair. 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: Your identification has been saved in /tmp/tmp8ogxqszp/key 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: Your public key has been saved in /tmp/tmp8ogxqszp/key.pub 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: The key fingerprint is: 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: SHA256:MoxJdU5HNPnG0Mt2tjVlIAilTgqE++X0u2UAcPjS+RE ceph-9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: The key's randomart image is: 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: +---[RSA 3072]----+ 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | .o.o +=*+. .. | 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | ...+ +Eo+... o| 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | .oo..+. = . ..| 2024-07-28T19:16:35.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | ...=*=. B o..| 2024-07-28T19:16:35.751 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | .o==oS. o o o.| 2024-07-28T19:16:35.751 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | . .oo. . | 2024-07-28T19:16:35.751 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | .o | 2024-07-28T19:16:35.751 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | .o | 2024-07-28T19:16:35.751 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: | .. | 2024-07-28T19:16:35.751 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:35 smithi100 bash[22098]: +----[SHA256]-----+ 2024-07-28T19:16:36.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:36 smithi100 bash[21828]: audit 2024-07-28T19:16:35.225948+0000 mgr.y (mgr.14118) 10 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:16:36.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:36 smithi100 bash[21828]: cephadm 2024-07-28T19:16:35.226199+0000 mgr.y (mgr.14118) 11 : cephadm [INF] Generating ssh key... 2024-07-28T19:16:36.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:36 smithi100 bash[21828]: audit 2024-07-28T19:16:35.365898+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:36.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:36 smithi100 bash[21828]: audit 2024-07-28T19:16:35.370809+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:37.842 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8zCVVX3jQecfc1iEgtBmGH4fmoaCsaHdTaw56VrMBKVOESCyLY0/k1vMvpmFFRv9OxonEnSQZenTtagJoo+8ZXF6h7EXHiYFD3BZyN7WaAKR/uvGAr2w1WaRWha++Yblg6UCSX0ssjvhvH+G/z+6h9vaamkABbutL3+S2BlFIeHi2jhabdJHo4qeqX53q6JQeL90kcoV2u2suZUIb4EgzyW26RAmIx/Z5GC4wZbymKB3wP9bq7g0T5yHh92gvaIKwgmQwQEeYbatu5E2NQBmz17+ORD5Uf7ADlGKWTVfn4epO82x4xomVpqz/mIv2tyroSstr77u0jkUQ+1qVjg1cDGnw33JUkIvWX/ZNmPDCF/gbP6VQN8cQv9RoGKRCMtLQQlUJkrwDCUbIfkQiie0/ksiiFRpqOHj9D6rMUqeNDQxoGPYNHkzIcWQwSCCFO8beiVNHhy61/RJ7GaVFnNitblR0e4ZOdVnKJmYoOGV1XqC+h+KUpCrZTPyo0N7m3AU= ceph-9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:16:37.842 INFO:teuthology.orchestra.run.smithi100.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-07-28T19:16:37.842 INFO:teuthology.orchestra.run.smithi100.stdout:Adding key to root@localhost authorized_keys... 2024-07-28T19:16:37.842 INFO:teuthology.orchestra.run.smithi100.stdout:Adding host smithi100... 2024-07-28T19:16:38.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:38 smithi100 bash[21828]: audit 2024-07-28T19:16:37.262511+0000 mgr.y (mgr.14118) 12 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:16:40.172 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:40 smithi100 bash[21828]: audit 2024-07-28T19:16:39.299138+0000 mgr.y (mgr.14118) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi100", "addr": "172.21.15.100", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:16:41.241 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:41 smithi100 bash[21828]: cephadm 2024-07-28T19:16:40.309843+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Deploying cephadm binary to smithi100 2024-07-28T19:16:43.335 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout Added host 'smithi100' with addr '172.21.15.100' 2024-07-28T19:16:43.335 INFO:teuthology.orchestra.run.smithi100.stdout:Deploying unmanaged mon service... 2024-07-28T19:16:43.601 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:43 smithi100 bash[21828]: audit 2024-07-28T19:16:42.466242+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:43.601 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:43 smithi100 bash[21828]: cephadm 2024-07-28T19:16:42.466690+0000 mgr.y (mgr.14118) 15 : cephadm [INF] Added host smithi100 2024-07-28T19:16:43.601 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:43 smithi100 bash[21828]: audit 2024-07-28T19:16:42.467153+0000 mon.a (mon.0) 56 : audit [DBG] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:16:46.096 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-07-28T19:16:46.096 INFO:teuthology.orchestra.run.smithi100.stdout:Deploying unmanaged mgr service... 2024-07-28T19:16:46.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:46 smithi100 bash[21828]: audit 2024-07-28T19:16:45.493820+0000 mgr.y (mgr.14118) 16 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:16:46.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:46 smithi100 bash[21828]: cephadm 2024-07-28T19:16:45.495854+0000 mgr.y (mgr.14118) 17 : cephadm [INF] Saving service mon spec with placement count:5 2024-07-28T19:16:46.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:46 smithi100 bash[21828]: audit 2024-07-28T19:16:45.498818+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:46.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:46 smithi100 bash[21828]: audit 2024-07-28T19:16:45.683406+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:46.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:46 smithi100 bash[21828]: audit 2024-07-28T19:16:45.972270+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:48.213 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-07-28T19:16:48.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:48 smithi100 bash[21828]: audit 2024-07-28T19:16:47.620718+0000 mgr.y (mgr.14118) 18 : audit [DBG] from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:16:49.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:48 smithi100 bash[21828]: cephadm 2024-07-28T19:16:47.622744+0000 mgr.y (mgr.14118) 19 : cephadm [INF] Saving service mgr spec with placement count:2 2024-07-28T19:16:49.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:48 smithi100 bash[21828]: audit 2024-07-28T19:16:47.627456+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:49.925 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:49 smithi100 bash[21828]: cluster 2024-07-28T19:16:48.299573+0000 mgr.y (mgr.14118) 20 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:16:50.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:50 smithi100 bash[21828]: audit 2024-07-28T19:16:49.719293+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.100:0/850324500' entity='client.admin' 2024-07-28T19:16:52.284 INFO:teuthology.orchestra.run.smithi100.stdout:Enabling the dashboard module... 2024-07-28T19:16:52.393 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:52 smithi100 bash[21828]: cluster 2024-07-28T19:16:50.300038+0000 mgr.y (mgr.14118) 21 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:16:52.393 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:52 smithi100 bash[21828]: audit 2024-07-28T19:16:51.025126+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:52.393 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:52 smithi100 bash[21828]: audit 2024-07-28T19:16:51.565469+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.14118 172.21.15.100:0/3354553407' entity='mgr.y' 2024-07-28T19:16:52.393 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:52 smithi100 bash[21828]: audit 2024-07-28T19:16:51.640377+0000 mon.a (mon.0) 64 : audit [INF] from='client.? 172.21.15.100:0/608420549' entity='client.admin' 2024-07-28T19:16:54.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:54 smithi100 bash[21828]: cluster 2024-07-28T19:16:52.300439+0000 mgr.y (mgr.14118) 22 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:16:54.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:54 smithi100 bash[21828]: audit 2024-07-28T19:16:53.903528+0000 mon.a (mon.0) 65 : audit [INF] from='client.? 172.21.15.100:0/893007824' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-07-28T19:16:54.500 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:54 smithi100 bash[22098]: ignoring --setuser ceph since I am not root 2024-07-28T19:16:54.500 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:54 smithi100 bash[22098]: ignoring --setgroup ceph since I am not root 2024-07-28T19:16:54.500 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:54 smithi100 bash[22098]: debug 2024-07-28T19:16:54.184+0000 7fa91e9f0ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:16:55.351 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:55 smithi100 bash[21828]: audit 2024-07-28T19:16:54.034402+0000 mon.a (mon.0) 66 : audit [INF] from='client.? 172.21.15.100:0/893007824' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-07-28T19:16:55.351 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:55 smithi100 bash[21828]: cluster 2024-07-28T19:16:54.034482+0000 mon.a (mon.0) 67 : cluster [DBG] mgrmap e9: y(active, since 25s) 2024-07-28T19:16:55.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:55 smithi100 bash[22098]: debug 2024-07-28T19:16:55.348+0000 7fa91e9f0ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:16:55.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:55 smithi100 bash[22098]: debug 2024-07-28T19:16:55.412+0000 7fa91e9f0ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:16:55.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:55 smithi100 bash[22098]: debug 2024-07-28T19:16:55.560+0000 7fa91e9f0ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:16:56.487 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[22098]: debug 2024-07-28T19:16:56.192+0000 7fa91e9f0ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:16:56.487 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[22098]: debug 2024-07-28T19:16:56.260+0000 7fa91e9f0ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:16:56.487 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[22098]: debug 2024-07-28T19:16:56.324+0000 7fa91e9f0ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:16:56.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[21828]: audit 2024-07-28T19:16:56.341392+0000 mon.a (mon.0) 68 : audit [DBG] from='client.? 172.21.15.100:0/4035007727' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-07-28T19:16:56.749 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[22098]: debug 2024-07-28T19:16:56.484+0000 7fa91e9f0ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:16:56.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[22098]: debug 2024-07-28T19:16:56.560+0000 7fa91e9f0ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:16:56.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[22098]: debug 2024-07-28T19:16:56.624+0000 7fa91e9f0ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:16:56.966 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout { 2024-07-28T19:16:56.966 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-07-28T19:16:56.966 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "available": true, 2024-07-28T19:16:56.967 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-07-28T19:16:56.967 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-07-28T19:16:56.967 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout } 2024-07-28T19:16:56.967 INFO:teuthology.orchestra.run.smithi100.stdout:Waiting for the mgr to restart... 2024-07-28T19:16:56.967 INFO:teuthology.orchestra.run.smithi100.stdout:Waiting for mgr epoch 9... 2024-07-28T19:16:57.158 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[22098]: debug 2024-07-28T19:16:56.848+0000 7fa91e9f0ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:16:57.158 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:56 smithi100 bash[22098]: debug 2024-07-28T19:16:56.912+0000 7fa91e9f0ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:16:57.158 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:57 smithi100 bash[22098]: debug 2024-07-28T19:16:57.084+0000 7fa91e9f0ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:16:57.499 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:57 smithi100 bash[22098]: debug 2024-07-28T19:16:57.156+0000 7fa91e9f0ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:16:57.499 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:57 smithi100 bash[22098]: debug 2024-07-28T19:16:57.384+0000 7fa91e9f0ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:16:58.000 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:57 smithi100 bash[22098]: debug 2024-07-28T19:16:57.540+0000 7fa91e9f0ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:16:58.000 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:57 smithi100 bash[22098]: debug 2024-07-28T19:16:57.616+0000 7fa91e9f0ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:16:58.000 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:57 smithi100 bash[22098]: debug 2024-07-28T19:16:57.772+0000 7fa91e9f0ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:16:58.811 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:58 smithi100 bash[22098]: debug 2024-07-28T19:16:58.532+0000 7fa91e9f0ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:16:58.811 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:58 smithi100 bash[22098]: debug 2024-07-28T19:16:58.612+0000 7fa91e9f0ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:16:59.148 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:58 smithi100 bash[22098]: debug 2024-07-28T19:16:58.808+0000 7fa91e9f0ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:16:59.149 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:58 smithi100 bash[22098]: debug 2024-07-28T19:16:58.884+0000 7fa91e9f0ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:16:59.462 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[22098]: debug 2024-07-28T19:16:59.204+0000 7fa91e9f0ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:16:59.462 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[22098]: debug 2024-07-28T19:16:59.276+0000 7fa91e9f0ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:16:59.462 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: cluster 2024-07-28T19:16:59.283128+0000 mon.a (mon.0) 69 : cluster [INF] Active manager daemon y restarted 2024-07-28T19:16:59.462 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: cluster 2024-07-28T19:16:59.283377+0000 mon.a (mon.0) 70 : cluster [INF] Activating manager daemon y 2024-07-28T19:16:59.462 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: cluster 2024-07-28T19:16:59.291301+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2024-07-28T19:16:59.462 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: cluster 2024-07-28T19:16:59.291513+0000 mon.a (mon.0) 72 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00821963s) 2024-07-28T19:16:59.462 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: audit 2024-07-28T19:16:59.293281+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:16:59.462 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: audit 2024-07-28T19:16:59.294609+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:16:59.463 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: audit 2024-07-28T19:16:59.295508+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:16:59.463 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: audit 2024-07-28T19:16:59.295586+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:16:59.463 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: audit 2024-07-28T19:16:59.295651+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:16:59.463 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: cluster 2024-07-28T19:16:59.319197+0000 mon.a (mon.0) 78 : cluster [INF] Manager daemon y is now available 2024-07-28T19:16:59.463 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:16:59 smithi100 bash[21828]: audit 2024-07-28T19:16:59.333518+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:17:00.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[21828]: audit 2024-07-28T19:16:59.337676+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:17:00.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[21828]: audit 2024-07-28T19:16:59.353476+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:17:00.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[21828]: audit 2024-07-28T19:16:59.355052+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:17:00.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[21828]: cluster 2024-07-28T19:17:00.291447+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e11: y(active, since 1.00816s) 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: debug 2024-07-28T19:17:00.348+0000 7fa8cf55b640 -1 log_channel(cephadm) log [ERR] : [28/Jul/2024:19:17:00] ENGINE Error in HTTPServer.serve 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: Traceback (most recent call last): 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: self._connections.run(self.expiration_interval) 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: self._run(expiration_interval) 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: s = self.context.wrap_socket( 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:17:00.753 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: return self.sslsocket_class._create( 2024-07-28T19:17:00.754 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:17:00.754 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: self.do_handshake() 2024-07-28T19:17:00.754 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:17:00.754 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: self._sslobj.do_handshake() 2024-07-28T19:17:00.754 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:17:00 smithi100 bash[22098]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:17:00.940 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout { 2024-07-28T19:17:00.940 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-07-28T19:17:00.940 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout "initialized": true 2024-07-28T19:17:00.940 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout } 2024-07-28T19:17:00.940 INFO:teuthology.orchestra.run.smithi100.stdout:mgr epoch 9 is available 2024-07-28T19:17:00.941 INFO:teuthology.orchestra.run.smithi100.stdout:Generating a dashboard self-signed certificate... 2024-07-28T19:17:01.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: cephadm 2024-07-28T19:17:00.228881+0000 mgr.y (mgr.14150) 1 : cephadm [INF] [28/Jul/2024:19:17:00] ENGINE Bus STARTING 2024-07-28T19:17:01.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: audit 2024-07-28T19:17:00.292404+0000 mgr.y (mgr.14150) 2 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: audit 2024-07-28T19:17:00.302496+0000 mgr.y (mgr.14150) 3 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: cephadm 2024-07-28T19:17:00.349560+0000 mgr.y (mgr.14150) 4 : cephadm [INF] [28/Jul/2024:19:17:00] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: cephadm 2024-07-28T19:17:00.349712+0000 mgr.y (mgr.14150) 5 : cephadm [INF] [28/Jul/2024:19:17:00] ENGINE Bus STARTED 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: cephadm 2024-07-28T19:17:00.351294+0000 mgr.y (mgr.14150) 6 : cephadm [ERR] [28/Jul/2024:19:17:00] ENGINE Error in HTTPServer.serve 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: Traceback (most recent call last): 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: self._connections.run(self.expiration_interval) 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: self._run(expiration_interval) 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:17:01.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: s = self.context.wrap_socket( 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: return self.sslsocket_class._create( 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: self.do_handshake() 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: self._sslobj.do_handshake() 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:17:01.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:01 smithi100 bash[21828]: audit 2024-07-28T19:17:00.352954+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:02.639 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:02 smithi100 bash[21828]: cluster 2024-07-28T19:17:01.364867+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e12: y(active, since 2s) 2024-07-28T19:17:03.073 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-07-28T19:17:03.073 INFO:teuthology.orchestra.run.smithi100.stdout:Creating initial admin user... 2024-07-28T19:17:03.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:03 smithi100 bash[21828]: audit 2024-07-28T19:17:02.306785+0000 mgr.y (mgr.14150) 7 : audit [DBG] from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:17:03.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:03 smithi100 bash[21828]: audit 2024-07-28T19:17:02.418189+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:03.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:03 smithi100 bash[21828]: audit 2024-07-28T19:17:02.425429+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:05.462 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$8NNLiPIG6BC/jucvjoeaEOl0XjuFlz4h55R6PUlekkKCs/MXk4iCW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1722194224, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-07-28T19:17:05.462 INFO:teuthology.orchestra.run.smithi100.stdout:Fetching dashboard port number... 2024-07-28T19:17:06.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:05 smithi100 bash[21828]: audit 2024-07-28T19:17:04.626338+0000 mgr.y (mgr.14150) 8 : audit [DBG] from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:17:06.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:05 smithi100 bash[21828]: audit 2024-07-28T19:17:04.655974+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:06.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:05 smithi100 bash[21828]: audit 2024-07-28T19:17:04.865500+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:06.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:05 smithi100 bash[21828]: audit 2024-07-28T19:17:05.187667+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:07.429 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stdout 8443 2024-07-28T19:17:07.429 INFO:teuthology.orchestra.run.smithi100.stdout:firewalld does not appear to be present 2024-07-28T19:17:07.429 INFO:teuthology.orchestra.run.smithi100.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-07-28T19:17:07.432 INFO:teuthology.orchestra.run.smithi100.stdout:Ceph Dashboard is now available at: 2024-07-28T19:17:07.432 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:07.433 INFO:teuthology.orchestra.run.smithi100.stdout: URL: https://smithi100.front.sepia.ceph.com:8443/ 2024-07-28T19:17:07.433 INFO:teuthology.orchestra.run.smithi100.stdout: User: admin 2024-07-28T19:17:07.433 INFO:teuthology.orchestra.run.smithi100.stdout: Password: zmqo0v1vke 2024-07-28T19:17:07.433 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:07.433 INFO:teuthology.orchestra.run.smithi100.stdout:Saving cluster configuration to /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config directory 2024-07-28T19:17:07.434 INFO:teuthology.orchestra.run.smithi100.stdout:Enabling autotune for osd_memory_target 2024-07-28T19:17:07.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:07 smithi100 bash[21828]: cluster 2024-07-28T19:17:06.193561+0000 mon.a (mon.0) 91 : cluster [DBG] mgrmap e13: y(active, since 6s) 2024-07-28T19:17:07.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:07 smithi100 bash[21828]: audit 2024-07-28T19:17:06.888380+0000 mon.a (mon.0) 92 : audit [DBG] from='client.? 172.21.15.100:0/3344448059' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-07-28T19:17:12.250 INFO:teuthology.orchestra.run.smithi100.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout:Or, if you are only running a single cluster on this host: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout:Please consider enabling telemetry to help improve Ceph: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: ceph telemetry on 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout:For more information see: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:12.251 INFO:teuthology.orchestra.run.smithi100.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-07-28T19:17:12.252 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:12.252 INFO:teuthology.orchestra.run.smithi100.stdout:Bootstrap complete. 2024-07-28T19:17:12.304 INFO:tasks.cephadm:Fetching config... 2024-07-28T19:17:12.304 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:17:12.305 DEBUG:teuthology.orchestra.run.smithi100:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-07-28T19:17:12.311 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-07-28T19:17:12.311 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:17:12.311 DEBUG:teuthology.orchestra.run.smithi100:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-07-28T19:17:12.360 INFO:tasks.cephadm:Fetching mon keyring... 2024-07-28T19:17:12.360 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:17:12.360 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/keyring of=/dev/stdout 2024-07-28T19:17:12.416 INFO:tasks.cephadm:Fetching pub ssh key... 2024-07-28T19:17:12.416 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:17:12.416 DEBUG:teuthology.orchestra.run.smithi100:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-07-28T19:17:12.464 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-07-28T19:17:12.465 DEBUG:teuthology.orchestra.run.smithi100:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8zCVVX3jQecfc1iEgtBmGH4fmoaCsaHdTaw56VrMBKVOESCyLY0/k1vMvpmFFRv9OxonEnSQZenTtagJoo+8ZXF6h7EXHiYFD3BZyN7WaAKR/uvGAr2w1WaRWha++Yblg6UCSX0ssjvhvH+G/z+6h9vaamkABbutL3+S2BlFIeHi2jhabdJHo4qeqX53q6JQeL90kcoV2u2suZUIb4EgzyW26RAmIx/Z5GC4wZbymKB3wP9bq7g0T5yHh92gvaIKwgmQwQEeYbatu5E2NQBmz17+ORD5Uf7ADlGKWTVfn4epO82x4xomVpqz/mIv2tyroSstr77u0jkUQ+1qVjg1cDGnw33JUkIvWX/ZNmPDCF/gbP6VQN8cQv9RoGKRCMtLQQlUJkrwDCUbIfkQiie0/ksiiFRpqOHj9D6rMUqeNDQxoGPYNHkzIcWQwSCCFO8beiVNHhy61/RJ7GaVFnNitblR0e4ZOdVnKJmYoOGV1XqC+h+KUpCrZTPyo0N7m3AU= ceph-9170a172-4d15-11ef-bcc8-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-07-28T19:17:12.529 INFO:teuthology.orchestra.run.smithi100.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8zCVVX3jQecfc1iEgtBmGH4fmoaCsaHdTaw56VrMBKVOESCyLY0/k1vMvpmFFRv9OxonEnSQZenTtagJoo+8ZXF6h7EXHiYFD3BZyN7WaAKR/uvGAr2w1WaRWha++Yblg6UCSX0ssjvhvH+G/z+6h9vaamkABbutL3+S2BlFIeHi2jhabdJHo4qeqX53q6JQeL90kcoV2u2suZUIb4EgzyW26RAmIx/Z5GC4wZbymKB3wP9bq7g0T5yHh92gvaIKwgmQwQEeYbatu5E2NQBmz17+ORD5Uf7ADlGKWTVfn4epO82x4xomVpqz/mIv2tyroSstr77u0jkUQ+1qVjg1cDGnw33JUkIvWX/ZNmPDCF/gbP6VQN8cQv9RoGKRCMtLQQlUJkrwDCUbIfkQiie0/ksiiFRpqOHj9D6rMUqeNDQxoGPYNHkzIcWQwSCCFO8beiVNHhy61/RJ7GaVFnNitblR0e4ZOdVnKJmYoOGV1XqC+h+KUpCrZTPyo0N7m3AU= ceph-9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:17:12.539 DEBUG:teuthology.orchestra.run.smithi138:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8zCVVX3jQecfc1iEgtBmGH4fmoaCsaHdTaw56VrMBKVOESCyLY0/k1vMvpmFFRv9OxonEnSQZenTtagJoo+8ZXF6h7EXHiYFD3BZyN7WaAKR/uvGAr2w1WaRWha++Yblg6UCSX0ssjvhvH+G/z+6h9vaamkABbutL3+S2BlFIeHi2jhabdJHo4qeqX53q6JQeL90kcoV2u2suZUIb4EgzyW26RAmIx/Z5GC4wZbymKB3wP9bq7g0T5yHh92gvaIKwgmQwQEeYbatu5E2NQBmz17+ORD5Uf7ADlGKWTVfn4epO82x4xomVpqz/mIv2tyroSstr77u0jkUQ+1qVjg1cDGnw33JUkIvWX/ZNmPDCF/gbP6VQN8cQv9RoGKRCMtLQQlUJkrwDCUbIfkQiie0/ksiiFRpqOHj9D6rMUqeNDQxoGPYNHkzIcWQwSCCFO8beiVNHhy61/RJ7GaVFnNitblR0e4ZOdVnKJmYoOGV1XqC+h+KUpCrZTPyo0N7m3AU= ceph-9170a172-4d15-11ef-bcc8-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-07-28T19:17:12.563 INFO:teuthology.orchestra.run.smithi138.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8zCVVX3jQecfc1iEgtBmGH4fmoaCsaHdTaw56VrMBKVOESCyLY0/k1vMvpmFFRv9OxonEnSQZenTtagJoo+8ZXF6h7EXHiYFD3BZyN7WaAKR/uvGAr2w1WaRWha++Yblg6UCSX0ssjvhvH+G/z+6h9vaamkABbutL3+S2BlFIeHi2jhabdJHo4qeqX53q6JQeL90kcoV2u2suZUIb4EgzyW26RAmIx/Z5GC4wZbymKB3wP9bq7g0T5yHh92gvaIKwgmQwQEeYbatu5E2NQBmz17+ORD5Uf7ADlGKWTVfn4epO82x4xomVpqz/mIv2tyroSstr77u0jkUQ+1qVjg1cDGnw33JUkIvWX/ZNmPDCF/gbP6VQN8cQv9RoGKRCMtLQQlUJkrwDCUbIfkQiie0/ksiiFRpqOHj9D6rMUqeNDQxoGPYNHkzIcWQwSCCFO8beiVNHhy61/RJ7GaVFnNitblR0e4ZOdVnKJmYoOGV1XqC+h+KUpCrZTPyo0N7m3AU= ceph-9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:17:12.573 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-07-28T19:17:12.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:12 smithi100 bash[21828]: audit 2024-07-28T19:17:11.593573+0000 mon.a (mon.0) 93 : audit [INF] from='client.? 172.21.15.100:0/3635762355' entity='client.admin' 2024-07-28T19:17:16.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:16 smithi100 bash[21828]: audit 2024-07-28T19:17:15.160027+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:16.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:16 smithi100 bash[21828]: audit 2024-07-28T19:17:15.163807+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:16.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:16 smithi100 bash[21828]: audit 2024-07-28T19:17:15.165013+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi100", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:17:16.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:16 smithi100 bash[21828]: audit 2024-07-28T19:17:15.172069+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:16.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:16 smithi100 bash[21828]: audit 2024-07-28T19:17:15.202566+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:17:16.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:16 smithi100 bash[21828]: audit 2024-07-28T19:17:15.209670+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:18.261 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:17:20.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:20 smithi100 bash[21828]: cluster 2024-07-28T19:17:19.297316+0000 mgr.y (mgr.14150) 9 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:22.134 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-07-28T19:17:22.134 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-07-28T19:17:23.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:22 smithi100 bash[21828]: cluster 2024-07-28T19:17:21.297646+0000 mgr.y (mgr.14150) 10 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:23.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:22 smithi100 bash[21828]: audit 2024-07-28T19:17:21.524032+0000 mon.a (mon.0) 100 : audit [INF] from='client.? 172.21.15.100:0/3981393155' entity='client.admin' 2024-07-28T19:17:23.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:22 smithi100 bash[21828]: audit 2024-07-28T19:17:21.534063+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:17:23.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:22 smithi100 bash[21828]: audit 2024-07-28T19:17:21.540523+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:25.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:24 smithi100 bash[21828]: cluster 2024-07-28T19:17:23.298060+0000 mgr.y (mgr.14150) 11 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:26.847 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:17:26.938 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:26 smithi100 bash[21828]: cluster 2024-07-28T19:17:25.298442+0000 mgr.y (mgr.14150) 12 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:29.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:28 smithi100 bash[21828]: cluster 2024-07-28T19:17:27.298857+0000 mgr.y (mgr.14150) 13 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:30.541 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi138 2024-07-28T19:17:30.541 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:17:30.541 DEBUG:teuthology.orchestra.run.smithi138:> dd of=/etc/ceph/ceph.conf 2024-07-28T19:17:30.548 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:17:30.548 DEBUG:teuthology.orchestra.run.smithi138:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:17:30.596 INFO:tasks.cephadm:Adding host smithi138 to orchestrator... 2024-07-28T19:17:30.596 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch host add smithi138 2024-07-28T19:17:31.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: cluster 2024-07-28T19:17:29.299210+0000 mgr.y (mgr.14150) 14 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: audit 2024-07-28T19:17:29.934311+0000 mgr.y (mgr.14150) 15 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: audit 2024-07-28T19:17:29.939960+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: audit 2024-07-28T19:17:29.941181+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: audit 2024-07-28T19:17:29.944027+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: audit 2024-07-28T19:17:29.944919+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: cephadm 2024-07-28T19:17:29.946064+0000 mgr.y (mgr.14150) 16 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: cephadm 2024-07-28T19:17:30.028069+0000 mgr.y (mgr.14150) 17 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: audit 2024-07-28T19:17:30.282610+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: audit 2024-07-28T19:17:30.287603+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:31.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:30 smithi100 bash[21828]: audit 2024-07-28T19:17:30.293714+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:32.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:31 smithi100 bash[21828]: cephadm 2024-07-28T19:17:30.109115+0000 mgr.y (mgr.14150) 18 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:17:32.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:31 smithi100 bash[21828]: cephadm 2024-07-28T19:17:30.196392+0000 mgr.y (mgr.14150) 19 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:17:33.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:32 smithi100 bash[21828]: cluster 2024-07-28T19:17:31.299618+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:35.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:34 smithi100 bash[21828]: cluster 2024-07-28T19:17:33.299917+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:35.310 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:17:37.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:36 smithi100 bash[21828]: cluster 2024-07-28T19:17:35.300277+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:39.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:38 smithi100 bash[21828]: cluster 2024-07-28T19:17:37.300659+0000 mgr.y (mgr.14150) 23 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:40.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:39 smithi100 bash[21828]: audit 2024-07-28T19:17:38.560746+0000 mgr.y (mgr.14150) 24 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi138", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:17:41.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:40 smithi100 bash[21828]: cluster 2024-07-28T19:17:39.301089+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:41.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:40 smithi100 bash[21828]: cephadm 2024-07-28T19:17:39.590328+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Deploying cephadm binary to smithi138 2024-07-28T19:17:41.830 INFO:teuthology.orchestra.run.smithi100.stdout:Added host 'smithi138' with addr '172.21.15.138' 2024-07-28T19:17:42.523 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch host ls --format=json 2024-07-28T19:17:43.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:42 smithi100 bash[21828]: cluster 2024-07-28T19:17:41.301473+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:43.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:42 smithi100 bash[21828]: audit 2024-07-28T19:17:41.828128+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:43.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:42 smithi100 bash[21828]: cephadm 2024-07-28T19:17:41.828903+0000 mgr.y (mgr.14150) 28 : cephadm [INF] Added host smithi138 2024-07-28T19:17:43.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:42 smithi100 bash[21828]: audit 2024-07-28T19:17:41.829761+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:17:43.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:42 smithi100 bash[21828]: audit 2024-07-28T19:17:42.091411+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:44.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:44 smithi100 bash[21828]: cluster 2024-07-28T19:17:43.301919+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:44.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:44 smithi100 bash[21828]: audit 2024-07-28T19:17:43.370468+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:44.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:44 smithi100 bash[21828]: audit 2024-07-28T19:17:44.013153+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:46.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:46 smithi100 bash[21828]: cluster 2024-07-28T19:17:45.302323+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:47.237 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:17:48.666 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:48 smithi100 bash[21828]: cluster 2024-07-28T19:17:47.302742+0000 mgr.y (mgr.14150) 31 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:50.308 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:17:50.308 INFO:teuthology.orchestra.run.smithi100.stdout:[{"addr": "172.21.15.100", "hostname": "smithi100", "labels": [], "status": ""}, {"addr": "172.21.15.138", "hostname": "smithi138", "labels": [], "status": ""}] 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: cluster 2024-07-28T19:17:49.303165+0000 mgr.y (mgr.14150) 32 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.313671+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.321294+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.327718+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.333753+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.334829+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi138", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.336698+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.338236+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: cephadm 2024-07-28T19:17:49.340364+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:17:50.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: cephadm 2024-07-28T19:17:49.512491+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:17:50.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: cephadm 2024-07-28T19:17:49.659473+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:17:50.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: cephadm 2024-07-28T19:17:49.793301+0000 mgr.y (mgr.14150) 36 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:17:50.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.926643+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:50.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.933006+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:50.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:50 smithi100 bash[21828]: audit 2024-07-28T19:17:49.940628+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:17:50.918 INFO:tasks.cephadm:Setting crush tunables to default 2024-07-28T19:17:50.918 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd crush tunables default 2024-07-28T19:17:51.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:51 smithi100 bash[21828]: audit 2024-07-28T19:17:50.307509+0000 mgr.y (mgr.14150) 37 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:17:52.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:52 smithi100 bash[21828]: cluster 2024-07-28T19:17:51.303547+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:54.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:54 smithi100 bash[21828]: cluster 2024-07-28T19:17:53.303929+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:55.618 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:17:56.698 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:56 smithi100 bash[21828]: cluster 2024-07-28T19:17:55.304303+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:58.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:58 smithi100 bash[21828]: cluster 2024-07-28T19:17:57.304674+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:17:59.336 INFO:teuthology.orchestra.run.smithi100.stderr:adjusted tunables profile to default 2024-07-28T19:17:59.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:17:59 smithi100 bash[21828]: audit 2024-07-28T19:17:58.638256+0000 mon.a (mon.0) 125 : audit [INF] from='client.? 172.21.15.100:0/2924663573' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-07-28T19:18:00.005 INFO:tasks.cephadm:Adding mon.a on smithi100 2024-07-28T19:18:00.005 INFO:tasks.cephadm:Adding mon.c on smithi100 2024-07-28T19:18:00.006 INFO:tasks.cephadm:Adding mon.b on smithi138 2024-07-28T19:18:00.006 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch apply mon '3;smithi100:172.21.15.100=a;smithi100:[v2:172.21.15.100:3301,v1:172.21.15.100:6790]=c;smithi138:172.21.15.138=b' 2024-07-28T19:18:00.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:00 smithi100 bash[21828]: cluster 2024-07-28T19:17:59.305085+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:00.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:00 smithi100 bash[21828]: audit 2024-07-28T19:17:59.335391+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.100:0/2924663573' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-07-28T19:18:00.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:00 smithi100 bash[21828]: cluster 2024-07-28T19:17:59.335511+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:01.215 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:02.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:02 smithi100 bash[21828]: cluster 2024-07-28T19:18:01.305493+0000 mgr.y (mgr.14150) 43 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:03.911 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled mon update... 2024-07-28T19:18:04.620 DEBUG:teuthology.orchestra.run.smithi100:mon.c> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.c.service 2024-07-28T19:18:04.622 DEBUG:teuthology.orchestra.run.smithi138:mon.b> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.b.service 2024-07-28T19:18:04.625 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-07-28T19:18:04.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph mon dump -f json 2024-07-28T19:18:05.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: cluster 2024-07-28T19:18:03.305870+0000 mgr.y (mgr.14150) 44 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: audit 2024-07-28T19:18:03.900010+0000 mgr.y (mgr.14150) 45 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi100:172.21.15.100=a;smithi100:[v2:172.21.15.100:3301,v1:172.21.15.100:6790]=c;smithi138:172.21.15.138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: cephadm 2024-07-28T19:18:03.905219+0000 mgr.y (mgr.14150) 46 : cephadm [INF] Saving service mon spec with placement smithi100:172.21.15.100=a;smithi100:[v2:172.21.15.100:3301,v1:172.21.15.100:6790]=c;smithi138:172.21.15.138=b;count:3 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: audit 2024-07-28T19:18:03.910579+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: audit 2024-07-28T19:18:03.912070+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: audit 2024-07-28T19:18:03.915343+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: audit 2024-07-28T19:18:03.917167+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: audit 2024-07-28T19:18:03.925078+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: audit 2024-07-28T19:18:03.928701+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: audit 2024-07-28T19:18:03.930449+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:05.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:04 smithi100 bash[21828]: cephadm 2024-07-28T19:18:03.932398+0000 mgr.y (mgr.14150) 47 : cephadm [INF] Deploying daemon mon.b on smithi138 2024-07-28T19:18:06.375 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:07.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:06 smithi100 bash[21828]: cluster 2024-07-28T19:18:05.306251+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:09.185 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:08 smithi138 systemd[1]: Started Ceph mon.b for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:18:09.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:08 smithi100 bash[21828]: cluster 2024-07-28T19:18:07.306638+0000 mgr.y (mgr.14150) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.275 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-07-28T19:18:10.275 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":1,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","modified":"2024-07-28T19:15:56.118341Z","created":"2024-07-28T19:15:56.118341Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:3300","nonce":0},{"type":"v1","addr":"172.21.15.100:6789","nonce":0}]},"addr":"172.21.15.100:6789/0","public_addr":"172.21.15.100:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-07-28T19:18:10.275 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 1 2024-07-28T19:18:10.312 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:10 smithi100 bash[21828]: audit 2024-07-28T19:18:09.019910+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.313 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:10 smithi100 bash[21828]: audit 2024-07-28T19:18:09.025956+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.313 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:10 smithi100 bash[21828]: audit 2024-07-28T19:18:09.032496+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.313 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:10 smithi100 bash[21828]: audit 2024-07-28T19:18:09.033663+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:10.313 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:10 smithi100 bash[21828]: audit 2024-07-28T19:18:09.035477+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:10.313 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:10 smithi100 bash[21828]: cephadm 2024-07-28T19:18:09.037456+0000 mgr.y (mgr.14150) 50 : cephadm [INF] Deploying daemon mon.c on smithi100 2024-07-28T19:18:10.560 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.326+0000 7f2c24bcd900 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-28T19:18:10.560 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.326+0000 7f2c24bcd900 0 ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable), process ceph-mon, pid 7 2024-07-28T19:18:10.560 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.326+0000 7f2c24bcd900 0 pidfile_write: ignore empty --pid-file 2024-07-28T19:18:10.560 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 0 load: jerasure load: lrc 2024-07-28T19:18:10.560 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: RocksDB version: 6.15.5 2024-07-28T19:18:10.560 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-07-28T19:18:10.560 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Compile date Jul 28 2024 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: DB SUMMARY 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: DB Session ID: WHTV433BF6OLVAALOL2J 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: CURRENT file: CURRENT 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: IDENTITY file: IDENTITY 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.error_if_exists: 0 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.create_if_missing: 0 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.paranoid_checks: 1 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.env: 0x56064fd17440 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.fs: Posix File System 2024-07-28T19:18:10.561 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.info_log: 0x560651336f40 2024-07-28T19:18:10.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-28T19:18:10.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.statistics: (nil) 2024-07-28T19:18:10.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.use_fsync: 0 2024-07-28T19:18:10.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_log_file_size: 0 2024-07-28T19:18:10.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-28T19:18:10.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.allow_fallocate: 1 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.use_direct_reads: 0 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.create_missing_column_families: 0 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.db_log_dir: 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-28T19:18:10.563 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.advise_random_on_open: 1 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.write_buffer_manager: 0x5606513f46f0 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.rate_limiter: (nil) 2024-07-28T19:18:10.564 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.unordered_write: 0 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.row_cache: None 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.wal_filter: None 2024-07-28T19:18:10.565 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.preserve_deletes: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.two_write_queues: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.manual_wal_flush: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.atomic_flush: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.log_readahead_size: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-28T19:18:10.566 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_background_jobs: 2 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_background_compactions: -1 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_subcompactions: 1 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_total_wal_size: 0 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_open_files: -1 2024-07-28T19:18:10.567 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bytes_per_sync: 0 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_background_flushes: -1 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Compression algorithms supported: 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: kZSTD supported: 0 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: kXpressCompression supported: 0 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: kBZip2Compression supported: 0 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: kLZ4Compression supported: 1 2024-07-28T19:18:10.568 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: kZlibCompression supported: 1 2024-07-28T19:18:10.569 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-28T19:18:10.569 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: kSnappyCompression supported: 1 2024-07-28T19:18:10.569 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-28T19:18:10.569 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-07-28T19:18:10.569 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-07-28T19:18:10.569 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-28T19:18:10.569 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.merge_operator: 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_filter: None 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_filter_factory: None 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5606512dfa58) 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cache_index_and_filter_blocks: 1 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: pin_top_level_index_and_filter: 1 2024-07-28T19:18:10.571 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: index_type: 0 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: data_block_index_type: 0 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: index_shortening: 1 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: data_block_hash_table_util_ratio: 0.750000 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: hash_index_allow_collision: 1 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: checksum: 1 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: no_block_cache: 0 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: block_cache: 0x5606513131f0 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: block_cache_name: BinnedLRUCache 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: block_cache_options: 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: capacity : 536870912 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: num_shard_bits : 4 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: strict_capacity_limit : 0 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: high_pri_pool_ratio: 0.000 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: block_cache_compressed: (nil) 2024-07-28T19:18:10.572 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: persistent_cache: (nil) 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: block_size: 4096 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: block_size_deviation: 10 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: block_restart_interval: 16 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: index_block_restart_interval: 1 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: metadata_block_size: 4096 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: partition_filters: 0 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: use_delta_encoding: 1 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: filter_policy: rocksdb.BuiltinBloomFilter 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: whole_key_filtering: 1 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: verify_compression: 0 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: read_amp_bytes_per_bit: 0 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: format_version: 4 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: enable_index_compression: 1 2024-07-28T19:18:10.573 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: block_align: 0 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compression: NoCompression 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.num_levels: 7 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-28T19:18:10.574 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compression_opts.level: 32767 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compression_opts.enabled: false 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-28T19:18:10.575 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.arena_block_size: 4194304 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-28T19:18:10.576 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-28T19:18:10.577 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.table_properties_collectors: 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.inplace_update_support: 0 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.bloom_locality: 0 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.max_successive_merges: 0 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.force_consistency_checks: 1 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.ttl: 2592000 2024-07-28T19:18:10.578 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.enable_blob_files: false 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.min_blob_size: 0 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.blob_file_size: 268435456 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194290334758, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-07-28T19:18:10.579 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194290335083, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1722194290, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b16564dc-38b9-4c58-b6f9-d6ea2c427c1b", "db_session_id": "WHTV433BF6OLVAALOL2J"}} 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194290335450, "job": 1, "event": "recovery_finished"} 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.330+0000 7f2c24bcd900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.334+0000 7f2c24bcd900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x560651348700 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.334+0000 7f2c24bcd900 4 rocksdb: DB pointer 0x560651316000 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.334+0000 7f2c1a999640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.334+0000 7f2c1a999640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ** DB Stats ** 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-28T19:18:10.580 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ** Compaction Stats [default] ** 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ** Compaction Stats [default] ** 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:18:10.581 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Cumulative compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Interval compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ** Compaction Stats [default] ** 2024-07-28T19:18:10.582 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ** Compaction Stats [default] ** 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:18:10.583 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Cumulative compaction: 0.00 GB write, 0.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.334+0000 7f2c24bcd900 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.334+0000 7f2c24bcd900 0 using public_addr v2:172.21.15.138:0/0 -> [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.334+0000 7f2c24bcd900 0 starting mon.b rank -1 at public addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] at bind addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.334+0000 7f2c24bcd900 1 mon.b@-1(???) e0 preinit fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.350+0000 7f2c1d99f640 0 mon.b@-1(synchronizing).mds e1 new map 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.350+0000 7f2c1d99f640 0 mon.b@-1(synchronizing).mds e1 print_map 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: e1 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: legacy client fscid: -1 2024-07-28T19:18:10.584 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: No filesystems configured 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.350+0000 7f2c1d99f640 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:31.299618+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:33.299917+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.585 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:35.300277+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:37.300659+0000 mgr.y (mgr.14150) 23 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:38.560746+0000 mgr.y (mgr.14150) 24 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi138", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:39.301089+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:17:39.590328+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Deploying cephadm binary to smithi138 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:41.301473+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:41.828128+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:17:41.828903+0000 mgr.y (mgr.14150) 28 : cephadm [INF] Added host smithi138 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:41.829761+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:42.091411+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:43.301919+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:43.370468+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:44.013153+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.586 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:45.302323+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:47.302742+0000 mgr.y (mgr.14150) 31 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:49.303165+0000 mgr.y (mgr.14150) 32 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.313671+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.321294+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.327718+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.333753+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.334829+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi138", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.336698+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.338236+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:17:49.340364+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:17:49.512491+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:17:49.659473+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:18:10.587 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:17:49.793301+0000 mgr.y (mgr.14150) 36 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.926643+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.933006+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:49.940628+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:50.307509+0000 mgr.y (mgr.14150) 37 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:51.303547+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:53.303929+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:55.304303+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:57.304674+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:58.638256+0000 mon.a (mon.0) 125 : audit [INF] from='client.? 172.21.15.100:0/2924663573' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:59.305085+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.588 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:17:59.335391+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.100:0/2924663573' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-07-28T19:18:10.589 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:17:59.335511+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:10.589 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:18:01.305493+0000 mgr.y (mgr.14150) 43 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.589 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:18:03.305870+0000 mgr.y (mgr.14150) 44 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.589 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:03.900010+0000 mgr.y (mgr.14150) 45 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi100:172.21.15.100=a;smithi100:[v2:172.21.15.100:3301,v1:172.21.15.100:6790]=c;smithi138:172.21.15.138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:10.589 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:18:03.905219+0000 mgr.y (mgr.14150) 46 : cephadm [INF] Saving service mon spec with placement smithi100:172.21.15.100=a;smithi100:[v2:172.21.15.100:3301,v1:172.21.15.100:6790]=c;smithi138:172.21.15.138=b;count:3 2024-07-28T19:18:10.589 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:03.910579+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.589 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:03.912070+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:10.590 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:03.915343+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:10.590 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:03.917167+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:10.590 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:03.925078+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.590 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:03.928701+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:10.590 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:03.930449+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:10.590 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:18:03.932398+0000 mgr.y (mgr.14150) 47 : cephadm [INF] Deploying daemon mon.b on smithi138 2024-07-28T19:18:10.591 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:18:05.306251+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.591 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cluster 2024-07-28T19:18:07.306638+0000 mgr.y (mgr.14150) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:10.591 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:09.019910+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.591 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:09.025956+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.591 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:09.032496+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:10.591 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:09.033663+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:10.591 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: audit 2024-07-28T19:18:09.035477+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:10.592 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: cephadm 2024-07-28T19:18:09.037456+0000 mgr.y (mgr.14150) 50 : cephadm [INF] Deploying daemon mon.c on smithi100 2024-07-28T19:18:10.592 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:10 smithi138 bash[23952]: debug 2024-07-28T19:18:10.354+0000 7f2c1d99f640 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-07-28T19:18:11.934 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-07-28T19:18:11.934 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph mon dump -f json 2024-07-28T19:18:12.917 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:18:12 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:12.917 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:12 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:12.917 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:12 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:13.231 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:18:13 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:13.231 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:13 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:13.231 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:13 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:13.231 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:13 smithi100 systemd[1]: Started Ceph mon.c for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:18:14.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.249+0000 7f8153080900 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-28T19:18:14.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.249+0000 7f8153080900 0 ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable), process ceph-mon, pid 7 2024-07-28T19:18:14.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.249+0000 7f8153080900 0 pidfile_write: ignore empty --pid-file 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 0 load: jerasure load: lrc 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: RocksDB version: 6.15.5 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Compile date Jul 28 2024 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: DB SUMMARY 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: DB Session ID: RY8GH6TLCN6W8WTDFOB3 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: CURRENT file: CURRENT 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: IDENTITY file: IDENTITY 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.error_if_exists: 0 2024-07-28T19:18:14.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.create_if_missing: 0 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.paranoid_checks: 1 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.env: 0x5595f2f5a440 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.fs: Posix File System 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.info_log: 0x5595f4b3af40 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.statistics: (nil) 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.use_fsync: 0 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_log_file_size: 0 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-28T19:18:14.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-28T19:18:14.504 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-28T19:18:14.504 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-28T19:18:14.504 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.allow_fallocate: 1 2024-07-28T19:18:14.504 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-28T19:18:14.504 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-28T19:18:14.504 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.use_direct_reads: 0 2024-07-28T19:18:14.504 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.create_missing_column_families: 0 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.db_log_dir: 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.advise_random_on_open: 1 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.write_buffer_manager: 0x5595f4bf8720 2024-07-28T19:18:14.505 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.rate_limiter: (nil) 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.unordered_write: 0 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-28T19:18:14.506 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.row_cache: None 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.wal_filter: None 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.preserve_deletes: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.two_write_queues: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.manual_wal_flush: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.atomic_flush: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.log_readahead_size: 0 2024-07-28T19:18:14.507 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_background_jobs: 2 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_background_compactions: -1 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_subcompactions: 1 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-28T19:18:14.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_total_wal_size: 0 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_open_files: -1 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bytes_per_sync: 0 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_background_flushes: -1 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Compression algorithms supported: 2024-07-28T19:18:14.509 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: kZSTD supported: 0 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: kXpressCompression supported: 0 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: kBZip2Compression supported: 0 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: kLZ4Compression supported: 1 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: kZlibCompression supported: 1 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: kSnappyCompression supported: 1 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-28T19:18:14.510 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.merge_operator: 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_filter: None 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_filter_factory: None 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5595f4ae3a58) 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cache_index_and_filter_blocks: 1 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-28T19:18:14.512 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: pin_top_level_index_and_filter: 1 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: index_type: 0 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: data_block_index_type: 0 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: index_shortening: 1 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: data_block_hash_table_util_ratio: 0.750000 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: hash_index_allow_collision: 1 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: checksum: 1 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: no_block_cache: 0 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: block_cache: 0x5595f4b171f0 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: block_cache_name: BinnedLRUCache 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: block_cache_options: 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: capacity : 536870912 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: num_shard_bits : 4 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: strict_capacity_limit : 0 2024-07-28T19:18:14.513 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: high_pri_pool_ratio: 0.000 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: block_cache_compressed: (nil) 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: persistent_cache: (nil) 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: block_size: 4096 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: block_size_deviation: 10 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: block_restart_interval: 16 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: index_block_restart_interval: 1 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: metadata_block_size: 4096 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: partition_filters: 0 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: use_delta_encoding: 1 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: filter_policy: rocksdb.BuiltinBloomFilter 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: whole_key_filtering: 1 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: verify_compression: 0 2024-07-28T19:18:14.514 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: read_amp_bytes_per_bit: 0 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: format_version: 4 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: enable_index_compression: 1 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: block_align: 0 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compression: NoCompression 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.num_levels: 7 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-28T19:18:14.515 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compression_opts.level: 32767 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:18:14.516 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compression_opts.enabled: false 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-28T19:18:14.517 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.arena_block_size: 4194304 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-28T19:18:14.518 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-28T19:18:14.519 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.table_properties_collectors: 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.inplace_update_support: 0 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.bloom_locality: 0 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.max_successive_merges: 0 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.force_consistency_checks: 1 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-28T19:18:14.520 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.ttl: 2592000 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.enable_blob_files: false 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.min_blob_size: 0 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.blob_file_size: 268435456 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.253+0000 7f8153080900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-07-28T19:18:14.521 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194294262382, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194294262733, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1722194294, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "8048c795-9f83-4f2e-a5ed-755167692c00", "db_session_id": "RY8GH6TLCN6W8WTDFOB3"}} 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722194294263183, "job": 1, "event": "recovery_finished"} 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5595f4b4c700 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 4 rocksdb: DB pointer 0x5595f4b1a000 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8148e4c640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-07-28T19:18:14.522 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8148e4c640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ** DB Stats ** 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ** Compaction Stats [default] ** 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:18:14.523 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ** Compaction Stats [default] ** 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:18:14.524 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Cumulative compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Interval compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ** Compaction Stats [default] ** 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:14.525 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ** Compaction Stats [default] ** 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Flush(GB): cumulative 0.000, interval 0.000 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:18:14.526 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.257+0000 7f8153080900 0 using public_addrv [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.261+0000 7f8153080900 0 starting mon.c rank -1 at public addrs [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] at bind addrs [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.261+0000 7f8153080900 1 mon.c@-1(???) e0 preinit fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 0 mon.c@-1(synchronizing).mds e1 new map 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 0 mon.c@-1(synchronizing).mds e1 print_map 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: e1 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-28T19:18:14.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: legacy client fscid: -1 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: No filesystems configured 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.281+0000 7f814be52640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-07-28T19:18:14.528 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:31.299618+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:33.299917+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:35.300277+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:37.300659+0000 mgr.y (mgr.14150) 23 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:38.560746+0000 mgr.y (mgr.14150) 24 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi138", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:39.301089+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:17:39.590328+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Deploying cephadm binary to smithi138 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:41.301473+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:41.828128+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:17:41.828903+0000 mgr.y (mgr.14150) 28 : cephadm [INF] Added host smithi138 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:41.829761+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:42.091411+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:43.301919+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.529 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:43.370468+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:44.013153+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:45.302323+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:47.302742+0000 mgr.y (mgr.14150) 31 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:49.303165+0000 mgr.y (mgr.14150) 32 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.313671+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.321294+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.327718+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.333753+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.334829+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi138", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.336698+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.338236+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:14.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:17:49.340364+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:17:49.512491+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:17:49.659473+0000 mgr.y (mgr.14150) 35 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:17:49.793301+0000 mgr.y (mgr.14150) 36 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.926643+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.933006+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:49.940628+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:50.307509+0000 mgr.y (mgr.14150) 37 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:51.303547+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:53.303929+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:55.304303+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:57.304674+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:58.638256+0000 mon.a (mon.0) 125 : audit [INF] from='client.? 172.21.15.100:0/2924663573' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-07-28T19:18:14.531 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:59.305085+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:17:59.335391+0000 mon.a (mon.0) 126 : audit [INF] from='client.? 172.21.15.100:0/2924663573' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:17:59.335511+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:18:01.305493+0000 mgr.y (mgr.14150) 43 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:18:03.305870+0000 mgr.y (mgr.14150) 44 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:03.900010+0000 mgr.y (mgr.14150) 45 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi100:172.21.15.100=a;smithi100:[v2:172.21.15.100:3301,v1:172.21.15.100:6790]=c;smithi138:172.21.15.138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:18:03.905219+0000 mgr.y (mgr.14150) 46 : cephadm [INF] Saving service mon spec with placement smithi100:172.21.15.100=a;smithi100:[v2:172.21.15.100:3301,v1:172.21.15.100:6790]=c;smithi138:172.21.15.138=b;count:3 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:03.910579+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:03.912070+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:03.915343+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:03.917167+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:03.925078+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:03.928701+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:14.532 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:03.930449+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:18:03.932398+0000 mgr.y (mgr.14150) 47 : cephadm [INF] Deploying daemon mon.b on smithi138 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:18:05.306251+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cluster 2024-07-28T19:18:07.306638+0000 mgr.y (mgr.14150) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:09.019910+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:09.025956+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:09.032496+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:09.033663+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: audit 2024-07-28T19:18:09.035477+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: cephadm 2024-07-28T19:18:09.037456+0000 mgr.y (mgr.14150) 50 : cephadm [INF] Deploying daemon mon.c on smithi100 2024-07-28T19:18:14.533 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:14 smithi100 bash[31715]: debug 2024-07-28T19:18:14.285+0000 7f814be52640 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-07-28T19:18:15.502 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:10.373752+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:18:15.502 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:10.373894+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:10.374015+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:11.307426+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:11.369501+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:12.370040+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:12.371265+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:13.307882+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:13.369634+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:14.294060+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:14.370073+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:15.294713+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:15.503 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:15.370076+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:15.418504+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:15.424323+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:15.441955+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:15.442009+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:15.442262+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 76s) 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: cluster 2024-07-28T19:18:15.442497+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:15.449057+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:15.456140+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:15.463950+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:15.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:15 smithi138 bash[23952]: audit 2024-07-28T19:18:15.470957+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:15.636 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:18:15.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:10.373752+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:18:15.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:10.373894+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:10.374015+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-07-28T19:18:15.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:11.307426+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:11.369501+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:12.370040+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:12.371265+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:13.307882+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:13.369634+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:14.294060+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:14.370073+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:15.294713+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:15.370076+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:15.418504+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:15.424323+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:18:15.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:15.441955+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2024-07-28T19:18:15.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:15.442009+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:15.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:15.442262+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 76s) 2024-07-28T19:18:15.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: cluster 2024-07-28T19:18:15.442497+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2024-07-28T19:18:15.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:15.449057+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:15.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:15.456140+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:15.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:15.463950+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:15.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:15 smithi100 bash[21828]: audit 2024-07-28T19:18:15.470957+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:16.750 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:18:16 smithi100 bash[22098]: debug 2024-07-28T19:18:16.365+0000 7fa902f42640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-07-28T19:18:21.680 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-07-28T19:18:21.680 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":3,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","modified":"2024-07-28T19:18:16.295754Z","created":"2024-07-28T19:15:56.118341Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:3300","nonce":0},{"type":"v1","addr":"172.21.15.100:6789","nonce":0}]},"addr":"172.21.15.100:6789/0","public_addr":"172.21.15.100:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:3300","nonce":0},{"type":"v1","addr":"172.21.15.138:6789","nonce":0}]},"addr":"172.21.15.138:6789/0","public_addr":"172.21.15.138:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:3301","nonce":0},{"type":"v1","addr":"172.21.15.100:6790","nonce":0}]},"addr":"172.21.15.100:6790/0","public_addr":"172.21.15.100:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-07-28T19:18:21.680 INFO:teuthology.orchestra.run.smithi138.stderr:dumped monmap epoch 3 2024-07-28T19:18:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:16.302302+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:18:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:16.302704+0000 mon.a (mon.0) 165 : cluster [INF] mon.a calling monitor election 2024-07-28T19:18:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:16.305660+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-28T19:18:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:16.322789+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:16.323183+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:17.295449+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:17.308672+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:18.294511+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:18.295849+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:19.295723+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:19.309122+0000 mgr.y (mgr.14150) 56 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:20.295119+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:21.295868+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:21.342794+0000 mon.a (mon.0) 173 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:21.349817+0000 mon.a (mon.0) 174 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:21.369889+0000 mon.a (mon.0) 175 : cluster [DBG] fsmap 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:21.369942+0000 mon.a (mon.0) 176 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:21.370193+0000 mon.a (mon.0) 177 : cluster [DBG] mgrmap e13: y(active, since 82s) 2024-07-28T19:18:21.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: cluster 2024-07-28T19:18:21.370428+0000 mon.a (mon.0) 178 : cluster [INF] overall HEALTH_OK 2024-07-28T19:18:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:21.377488+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:21.382790+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:21.388554+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:21.394088+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.693 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:21 smithi138 bash[23952]: audit 2024-07-28T19:18:21.399648+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:10.373752+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:18:21.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:10.373894+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:10.374015+0000 mon.a (mon.0) 144 : cluster [INF] mon.a calling monitor election 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:11.307426+0000 mgr.y (mgr.14150) 52 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:11.369501+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:12.370040+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:12.371265+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:13.307882+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:13.369634+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:14.294060+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:14.370073+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:15.294713+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:15.370076+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:15.418504+0000 mon.a (mon.0) 152 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-28T19:18:21.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:15.424323+0000 mon.a (mon.0) 153 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:15.441955+0000 mon.a (mon.0) 154 : cluster [DBG] fsmap 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:15.442009+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:15.442262+0000 mon.a (mon.0) 156 : cluster [DBG] mgrmap e13: y(active, since 76s) 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:15.442497+0000 mon.a (mon.0) 157 : cluster [INF] overall HEALTH_OK 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:15.449057+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:15.456140+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:15.463950+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:15.470957+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:16.302302+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:16.302704+0000 mon.a (mon.0) 165 : cluster [INF] mon.a calling monitor election 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:16.305660+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:16.322789+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:16.323183+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:17.295449+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:17.308672+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:18.294511+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:18.295849+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:19.295723+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:19.309122+0000 mgr.y (mgr.14150) 56 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:20.295119+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:21.295868+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:21.342794+0000 mon.a (mon.0) 173 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:21.349817+0000 mon.a (mon.0) 174 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:21.369889+0000 mon.a (mon.0) 175 : cluster [DBG] fsmap 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:21.369942+0000 mon.a (mon.0) 176 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:21.370193+0000 mon.a (mon.0) 177 : cluster [DBG] mgrmap e13: y(active, since 82s) 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: cluster 2024-07-28T19:18:21.370428+0000 mon.a (mon.0) 178 : cluster [INF] overall HEALTH_OK 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:21.377488+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:21.382790+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:21.388554+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.754 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:21.394088+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.754 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[31715]: audit 2024-07-28T19:18:21.399648+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.754 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:16.302302+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:18:21.754 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:16.302704+0000 mon.a (mon.0) 165 : cluster [INF] mon.a calling monitor election 2024-07-28T19:18:21.754 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:16.305660+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-28T19:18:21.754 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:16.322789+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:18:21.754 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:16.323183+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:17.295449+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:17.308672+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:18.294511+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:18.295849+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:19.295723+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:19.309122+0000 mgr.y (mgr.14150) 56 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:20.295119+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:21.295868+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:21.755 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:21.342794+0000 mon.a (mon.0) 173 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:21.349817+0000 mon.a (mon.0) 174 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:21.369889+0000 mon.a (mon.0) 175 : cluster [DBG] fsmap 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:21.369942+0000 mon.a (mon.0) 176 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:21.370193+0000 mon.a (mon.0) 177 : cluster [DBG] mgrmap e13: y(active, since 82s) 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: cluster 2024-07-28T19:18:21.370428+0000 mon.a (mon.0) 178 : cluster [INF] overall HEALTH_OK 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:21.377488+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:21.382790+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:21.388554+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:21.394088+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:21.756 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:21 smithi100 bash[21828]: audit 2024-07-28T19:18:21.399648+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.314 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-07-28T19:18:22.315 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph config generate-minimal-conf 2024-07-28T19:18:22.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: cluster 2024-07-28T19:18:21.309469+0000 mgr.y (mgr.14150) 57 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:21.679586+0000 mon.a (mon.0) 184 : audit [DBG] from='client.? 172.21.15.138:0/1882398638' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:21.818681+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:21.820569+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: cephadm 2024-07-28T19:18:21.822836+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: cephadm 2024-07-28T19:18:21.823300+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: cephadm 2024-07-28T19:18:21.956860+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: cephadm 2024-07-28T19:18:21.960483+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.075250+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.080319+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.091427+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.097385+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.103443+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.137335+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:22.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.138595+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.139663+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[21828]: audit 2024-07-28T19:18:22.295935+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: cluster 2024-07-28T19:18:21.309469+0000 mgr.y (mgr.14150) 57 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:21.679586+0000 mon.a (mon.0) 184 : audit [DBG] from='client.? 172.21.15.138:0/1882398638' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:21.818681+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:21.820569+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: cephadm 2024-07-28T19:18:21.822836+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: cephadm 2024-07-28T19:18:21.823300+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: cephadm 2024-07-28T19:18:21.956860+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: cephadm 2024-07-28T19:18:21.960483+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.075250+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.080319+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.091427+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.097385+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.103443+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.137335+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:22.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.138595+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:22.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.139663+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:22.753 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:22 smithi100 bash[31715]: audit 2024-07-28T19:18:22.295935+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:22.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: cluster 2024-07-28T19:18:21.309469+0000 mgr.y (mgr.14150) 57 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:21.679586+0000 mon.a (mon.0) 184 : audit [DBG] from='client.? 172.21.15.138:0/1882398638' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:21.818681+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:21.820569+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: cephadm 2024-07-28T19:18:21.822836+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: cephadm 2024-07-28T19:18:21.823300+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: cephadm 2024-07-28T19:18:21.956860+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: cephadm 2024-07-28T19:18:21.960483+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.075250+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.080319+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.091427+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.097385+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.811 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.103443+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:22.811 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.137335+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:22.811 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.138595+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:22.811 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.139663+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:22.811 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:22 smithi138 bash[23952]: audit 2024-07-28T19:18:22.295935+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:18:23.701 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:23 smithi100 bash[21828]: cephadm 2024-07-28T19:18:22.136769+0000 mgr.y (mgr.14150) 62 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-07-28T19:18:23.701 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:23 smithi100 bash[21828]: cephadm 2024-07-28T19:18:22.140756+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring daemon mon.a on smithi100 2024-07-28T19:18:23.701 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:23 smithi100 bash[31715]: cephadm 2024-07-28T19:18:22.136769+0000 mgr.y (mgr.14150) 62 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-07-28T19:18:23.701 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:23 smithi100 bash[31715]: cephadm 2024-07-28T19:18:22.140756+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring daemon mon.a on smithi100 2024-07-28T19:18:23.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:23 smithi138 bash[23952]: cephadm 2024-07-28T19:18:22.136769+0000 mgr.y (mgr.14150) 62 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-07-28T19:18:23.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:23 smithi138 bash[23952]: cephadm 2024-07-28T19:18:22.140756+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring daemon mon.a on smithi100 2024-07-28T19:18:24.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[21828]: cluster 2024-07-28T19:18:23.309834+0000 mgr.y (mgr.14150) 64 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:24.796 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[21828]: audit 2024-07-28T19:18:24.352468+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:24.796 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[21828]: audit 2024-07-28T19:18:24.357309+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:24.796 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[21828]: audit 2024-07-28T19:18:24.358729+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:24.796 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[21828]: audit 2024-07-28T19:18:24.359738+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:24.796 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[21828]: audit 2024-07-28T19:18:24.360728+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:24.797 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[31715]: cluster 2024-07-28T19:18:23.309834+0000 mgr.y (mgr.14150) 64 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:24.797 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[31715]: audit 2024-07-28T19:18:24.352468+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:24.797 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[31715]: audit 2024-07-28T19:18:24.357309+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:24.797 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[31715]: audit 2024-07-28T19:18:24.358729+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:24.797 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[31715]: audit 2024-07-28T19:18:24.359738+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:24.798 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:24 smithi100 bash[31715]: audit 2024-07-28T19:18:24.360728+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:24.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:24 smithi138 bash[23952]: cluster 2024-07-28T19:18:23.309834+0000 mgr.y (mgr.14150) 64 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:24.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:24 smithi138 bash[23952]: audit 2024-07-28T19:18:24.352468+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:24.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:24 smithi138 bash[23952]: audit 2024-07-28T19:18:24.357309+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:24.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:24 smithi138 bash[23952]: audit 2024-07-28T19:18:24.358729+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:24.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:24 smithi138 bash[23952]: audit 2024-07-28T19:18:24.359738+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:24.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:24 smithi138 bash[23952]: audit 2024-07-28T19:18:24.360728+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:25.721 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:25 smithi100 bash[21828]: cephadm 2024-07-28T19:18:24.358329+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-28T19:18:25.721 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:25 smithi100 bash[21828]: cephadm 2024-07-28T19:18:24.362098+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi100 2024-07-28T19:18:25.721 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:25 smithi100 bash[31715]: cephadm 2024-07-28T19:18:24.358329+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-28T19:18:25.721 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:25 smithi100 bash[31715]: cephadm 2024-07-28T19:18:24.362098+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi100 2024-07-28T19:18:25.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:25 smithi138 bash[23952]: cephadm 2024-07-28T19:18:24.358329+0000 mgr.y (mgr.14150) 65 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-28T19:18:25.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:25 smithi138 bash[23952]: cephadm 2024-07-28T19:18:24.362098+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring daemon mon.c on smithi100 2024-07-28T19:18:26.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[21828]: cluster 2024-07-28T19:18:25.310184+0000 mgr.y (mgr.14150) 67 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[21828]: audit 2024-07-28T19:18:26.290075+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[21828]: audit 2024-07-28T19:18:26.298180+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[21828]: audit 2024-07-28T19:18:26.299722+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[21828]: audit 2024-07-28T19:18:26.300790+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[21828]: audit 2024-07-28T19:18:26.301720+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[31715]: cluster 2024-07-28T19:18:25.310184+0000 mgr.y (mgr.14150) 67 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[31715]: audit 2024-07-28T19:18:26.290075+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[31715]: audit 2024-07-28T19:18:26.298180+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:26.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[31715]: audit 2024-07-28T19:18:26.299722+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:26.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[31715]: audit 2024-07-28T19:18:26.300790+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:26.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:26 smithi100 bash[31715]: audit 2024-07-28T19:18:26.301720+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:26.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:26 smithi138 bash[23952]: cluster 2024-07-28T19:18:25.310184+0000 mgr.y (mgr.14150) 67 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:26.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:26 smithi138 bash[23952]: audit 2024-07-28T19:18:26.290075+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:26.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:26 smithi138 bash[23952]: audit 2024-07-28T19:18:26.298180+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:26.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:26 smithi138 bash[23952]: audit 2024-07-28T19:18:26.299722+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:18:26.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:26 smithi138 bash[23952]: audit 2024-07-28T19:18:26.300790+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:18:26.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:26 smithi138 bash[23952]: audit 2024-07-28T19:18:26.301720+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:27.209 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:18:27.710 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:27 smithi138 bash[23952]: cephadm 2024-07-28T19:18:26.299311+0000 mgr.y (mgr.14150) 68 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-28T19:18:27.710 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:27 smithi138 bash[23952]: cephadm 2024-07-28T19:18:26.302748+0000 mgr.y (mgr.14150) 69 : cephadm [INF] Reconfiguring daemon mon.b on smithi138 2024-07-28T19:18:27.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:27 smithi100 bash[21828]: cephadm 2024-07-28T19:18:26.299311+0000 mgr.y (mgr.14150) 68 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-28T19:18:27.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:27 smithi100 bash[21828]: cephadm 2024-07-28T19:18:26.302748+0000 mgr.y (mgr.14150) 69 : cephadm [INF] Reconfiguring daemon mon.b on smithi138 2024-07-28T19:18:27.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:27 smithi100 bash[31715]: cephadm 2024-07-28T19:18:26.299311+0000 mgr.y (mgr.14150) 68 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-28T19:18:27.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:27 smithi100 bash[31715]: cephadm 2024-07-28T19:18:26.302748+0000 mgr.y (mgr.14150) 69 : cephadm [INF] Reconfiguring daemon mon.b on smithi138 2024-07-28T19:18:29.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[21828]: cluster 2024-07-28T19:18:27.310552+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:29.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[21828]: audit 2024-07-28T19:18:28.102293+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[21828]: audit 2024-07-28T19:18:28.109503+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[21828]: audit 2024-07-28T19:18:28.111326+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[21828]: audit 2024-07-28T19:18:28.114026+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[21828]: audit 2024-07-28T19:18:28.115392+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[21828]: audit 2024-07-28T19:18:28.123605+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[31715]: cluster 2024-07-28T19:18:27.310552+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[31715]: audit 2024-07-28T19:18:28.102293+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[31715]: audit 2024-07-28T19:18:28.109503+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[31715]: audit 2024-07-28T19:18:28.111326+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[31715]: audit 2024-07-28T19:18:28.114026+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:29.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[31715]: audit 2024-07-28T19:18:28.115392+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:29.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:29 smithi100 bash[31715]: audit 2024-07-28T19:18:28.123605+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:29.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:29 smithi138 bash[23952]: cluster 2024-07-28T19:18:27.310552+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:29.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:29 smithi138 bash[23952]: audit 2024-07-28T19:18:28.102293+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:29.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:29 smithi138 bash[23952]: audit 2024-07-28T19:18:28.109503+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:29.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:29 smithi138 bash[23952]: audit 2024-07-28T19:18:28.111326+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:29.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:29 smithi138 bash[23952]: audit 2024-07-28T19:18:28.114026+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:29.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:29 smithi138 bash[23952]: audit 2024-07-28T19:18:28.115392+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:29.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:29 smithi138 bash[23952]: audit 2024-07-28T19:18:28.123605+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:30.283 INFO:teuthology.orchestra.run.smithi100.stdout:# minimal ceph.conf for 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:18:30.283 INFO:teuthology.orchestra.run.smithi100.stdout:[global] 2024-07-28T19:18:30.283 INFO:teuthology.orchestra.run.smithi100.stdout: fsid = 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:18:30.283 INFO:teuthology.orchestra.run.smithi100.stdout: mon_host = [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] 2024-07-28T19:18:30.899 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-07-28T19:18:30.899 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:18:30.899 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/ceph/ceph.conf 2024-07-28T19:18:30.913 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:18:30.913 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:18:30.972 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:18:30.972 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.conf 2024-07-28T19:18:30.986 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:18:30.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:18:31.043 INFO:tasks.cephadm:Adding mgr.y on smithi100 2024-07-28T19:18:31.043 INFO:tasks.cephadm:Adding mgr.x on smithi138 2024-07-28T19:18:31.043 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch apply mgr '2;smithi100=y;smithi138=x' 2024-07-28T19:18:31.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:31 smithi100 bash[21828]: cluster 2024-07-28T19:18:29.310957+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:31.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:31 smithi100 bash[21828]: audit 2024-07-28T19:18:30.282805+0000 mon.a (mon.0) 212 : audit [DBG] from='client.? 172.21.15.100:0/939763941' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:31.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:31 smithi100 bash[31715]: cluster 2024-07-28T19:18:29.310957+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:31.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:31 smithi100 bash[31715]: audit 2024-07-28T19:18:30.282805+0000 mon.a (mon.0) 212 : audit [DBG] from='client.? 172.21.15.100:0/939763941' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:31.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:31 smithi138 bash[23952]: cluster 2024-07-28T19:18:29.310957+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:31 smithi138 bash[23952]: audit 2024-07-28T19:18:30.282805+0000 mon.a (mon.0) 212 : audit [DBG] from='client.? 172.21.15.100:0/939763941' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:33 smithi100 bash[21828]: cluster 2024-07-28T19:18:31.311370+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:33.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:33 smithi100 bash[31715]: cluster 2024-07-28T19:18:31.311370+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:33.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:33 smithi138 bash[23952]: cluster 2024-07-28T19:18:31.311370+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:34.787 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:18:35.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:35 smithi138 bash[23952]: cluster 2024-07-28T19:18:33.311762+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:35.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:35 smithi100 bash[21828]: cluster 2024-07-28T19:18:33.311762+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:35.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:35 smithi100 bash[31715]: cluster 2024-07-28T19:18:33.311762+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:36.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:36 smithi100 bash[21828]: cluster 2024-07-28T19:18:35.312111+0000 mgr.y (mgr.14150) 74 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:36.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:36 smithi100 bash[31715]: cluster 2024-07-28T19:18:35.312111+0000 mgr.y (mgr.14150) 74 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:36.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:36 smithi138 bash[23952]: cluster 2024-07-28T19:18:35.312111+0000 mgr.y (mgr.14150) 74 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:37.683 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled mgr update... 2024-07-28T19:18:38.567 DEBUG:teuthology.orchestra.run.smithi138:mgr.x> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.x.service 2024-07-28T19:18:38.570 INFO:tasks.cephadm:Deploying OSDs... 2024-07-28T19:18:38.570 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:18:38.570 DEBUG:teuthology.orchestra.run.smithi100:> dd if=/scratch_devs of=/dev/stdout 2024-07-28T19:18:38.577 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-07-28T19:18:38.577 DEBUG:teuthology.orchestra.run.smithi100:> stat /dev/vg_nvme/lv_1 2024-07-28T19:18:38.625 INFO:teuthology.orchestra.run.smithi100.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-07-28T19:18:38.626 INFO:teuthology.orchestra.run.smithi100.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-28T19:18:38.626 INFO:teuthology.orchestra.run.smithi100.stdout:Device: 5h/5d Inode: 879 Links: 1 2024-07-28T19:18:38.626 INFO:teuthology.orchestra.run.smithi100.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-28T19:18:38.626 INFO:teuthology.orchestra.run.smithi100.stdout:Access: 2024-07-28 19:09:13.947562002 +0000 2024-07-28T19:18:38.626 INFO:teuthology.orchestra.run.smithi100.stdout:Modify: 2024-07-28 19:09:13.719564039 +0000 2024-07-28T19:18:38.626 INFO:teuthology.orchestra.run.smithi100.stdout:Change: 2024-07-28 19:09:13.719564039 +0000 2024-07-28T19:18:38.626 INFO:teuthology.orchestra.run.smithi100.stdout: Birth: - 2024-07-28T19:18:38.626 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-07-28T19:18:38.681 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records in 2024-07-28T19:18:38.681 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records out 2024-07-28T19:18:38.681 INFO:teuthology.orchestra.run.smithi100.stderr:512 bytes copied, 0.000375004 s, 1.4 MB/s 2024-07-28T19:18:38.682 DEBUG:teuthology.orchestra.run.smithi100:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-07-28T19:18:38.730 DEBUG:teuthology.orchestra.run.smithi100:> stat /dev/vg_nvme/lv_2 2024-07-28T19:18:38.777 INFO:teuthology.orchestra.run.smithi100.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-07-28T19:18:38.777 INFO:teuthology.orchestra.run.smithi100.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-28T19:18:38.777 INFO:teuthology.orchestra.run.smithi100.stdout:Device: 5h/5d Inode: 893 Links: 1 2024-07-28T19:18:38.777 INFO:teuthology.orchestra.run.smithi100.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-28T19:18:38.777 INFO:teuthology.orchestra.run.smithi100.stdout:Access: 2024-07-28 19:09:14.407557893 +0000 2024-07-28T19:18:38.777 INFO:teuthology.orchestra.run.smithi100.stdout:Modify: 2024-07-28 19:09:14.175559966 +0000 2024-07-28T19:18:38.777 INFO:teuthology.orchestra.run.smithi100.stdout:Change: 2024-07-28 19:09:14.175559966 +0000 2024-07-28T19:18:38.777 INFO:teuthology.orchestra.run.smithi100.stdout: Birth: - 2024-07-28T19:18:38.778 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-07-28T19:18:38.831 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records in 2024-07-28T19:18:38.831 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records out 2024-07-28T19:18:38.831 INFO:teuthology.orchestra.run.smithi100.stderr:512 bytes copied, 0.000340442 s, 1.5 MB/s 2024-07-28T19:18:38.832 DEBUG:teuthology.orchestra.run.smithi100:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-07-28T19:18:38.882 DEBUG:teuthology.orchestra.run.smithi100:> stat /dev/vg_nvme/lv_3 2024-07-28T19:18:38.929 INFO:teuthology.orchestra.run.smithi100.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-07-28T19:18:38.929 INFO:teuthology.orchestra.run.smithi100.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-28T19:18:38.929 INFO:teuthology.orchestra.run.smithi100.stdout:Device: 5h/5d Inode: 908 Links: 1 2024-07-28T19:18:38.929 INFO:teuthology.orchestra.run.smithi100.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-28T19:18:38.929 INFO:teuthology.orchestra.run.smithi100.stdout:Access: 2024-07-28 19:09:14.879553677 +0000 2024-07-28T19:18:38.929 INFO:teuthology.orchestra.run.smithi100.stdout:Modify: 2024-07-28 19:09:14.647555749 +0000 2024-07-28T19:18:38.929 INFO:teuthology.orchestra.run.smithi100.stdout:Change: 2024-07-28 19:09:14.647555749 +0000 2024-07-28T19:18:38.929 INFO:teuthology.orchestra.run.smithi100.stdout: Birth: - 2024-07-28T19:18:38.929 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-07-28T19:18:38.981 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: cluster 2024-07-28T19:18:37.312489+0000 mgr.y (mgr.14150) 75 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:38.981 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.672296+0000 mgr.y (mgr.14150) 76 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi100=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:38.981 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: cephadm 2024-07-28T19:18:37.675266+0000 mgr.y (mgr.14150) 77 : cephadm [INF] Saving service mgr spec with placement smithi100=y;smithi138=x;count:2 2024-07-28T19:18:38.981 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.682135+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:38.981 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.683847+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.687145+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.688846+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.698398+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.701220+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.706725+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.708454+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: audit 2024-07-28T19:18:37.710205+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[21828]: cephadm 2024-07-28T19:18:37.712146+0000 mgr.y (mgr.14150) 78 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: cluster 2024-07-28T19:18:37.312489+0000 mgr.y (mgr.14150) 75 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:38.982 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.672296+0000 mgr.y (mgr.14150) 76 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi100=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: cephadm 2024-07-28T19:18:37.675266+0000 mgr.y (mgr.14150) 77 : cephadm [INF] Saving service mgr spec with placement smithi100=y;smithi138=x;count:2 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.682135+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.683847+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.687145+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.688846+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.698398+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.701220+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.706725+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.708454+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: audit 2024-07-28T19:18:37.710205+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:38.983 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:38 smithi100 bash[31715]: cephadm 2024-07-28T19:18:37.712146+0000 mgr.y (mgr.14150) 78 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:18:38.984 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records in 2024-07-28T19:18:38.984 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records out 2024-07-28T19:18:38.984 INFO:teuthology.orchestra.run.smithi100.stderr:512 bytes copied, 0.00044176 s, 1.2 MB/s 2024-07-28T19:18:38.985 DEBUG:teuthology.orchestra.run.smithi100:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-07-28T19:18:39.034 DEBUG:teuthology.orchestra.run.smithi100:> stat /dev/vg_nvme/lv_4 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: cluster 2024-07-28T19:18:37.312489+0000 mgr.y (mgr.14150) 75 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.672296+0000 mgr.y (mgr.14150) 76 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi100=y;smithi138=x", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: cephadm 2024-07-28T19:18:37.675266+0000 mgr.y (mgr.14150) 77 : cephadm [INF] Saving service mgr spec with placement smithi100=y;smithi138=x;count:2 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.682135+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.683847+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.687145+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.688846+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.698398+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.701220+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.706725+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-07-28T19:18:39.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.708454+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:18:39.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: audit 2024-07-28T19:18:37.710205+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:39.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:38 smithi138 bash[23952]: cephadm 2024-07-28T19:18:37.712146+0000 mgr.y (mgr.14150) 78 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:18:39.081 INFO:teuthology.orchestra.run.smithi100.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-07-28T19:18:39.081 INFO:teuthology.orchestra.run.smithi100.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-28T19:18:39.081 INFO:teuthology.orchestra.run.smithi100.stdout:Device: 5h/5d Inode: 932 Links: 1 2024-07-28T19:18:39.081 INFO:teuthology.orchestra.run.smithi100.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-28T19:18:39.081 INFO:teuthology.orchestra.run.smithi100.stdout:Access: 2024-07-28 19:09:15.387549140 +0000 2024-07-28T19:18:39.081 INFO:teuthology.orchestra.run.smithi100.stdout:Modify: 2024-07-28 19:09:15.139551355 +0000 2024-07-28T19:18:39.081 INFO:teuthology.orchestra.run.smithi100.stdout:Change: 2024-07-28 19:09:15.139551355 +0000 2024-07-28T19:18:39.082 INFO:teuthology.orchestra.run.smithi100.stdout: Birth: - 2024-07-28T19:18:39.082 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-07-28T19:18:39.135 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records in 2024-07-28T19:18:39.136 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records out 2024-07-28T19:18:39.136 INFO:teuthology.orchestra.run.smithi100.stderr:512 bytes copied, 0.000359572 s, 1.4 MB/s 2024-07-28T19:18:39.137 DEBUG:teuthology.orchestra.run.smithi100:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-07-28T19:18:39.186 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:18:39.187 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/scratch_devs of=/dev/stdout 2024-07-28T19:18:39.193 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-07-28T19:18:39.193 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_1 2024-07-28T19:18:39.239 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-07-28T19:18:39.240 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-28T19:18:39.240 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 5h/5d Inode: 891 Links: 1 2024-07-28T19:18:39.240 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-28T19:18:39.240 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2024-07-28 19:09:13.808990216 +0000 2024-07-28T19:18:39.240 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2024-07-28 19:09:13.544992575 +0000 2024-07-28T19:18:39.240 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2024-07-28 19:09:13.544992575 +0000 2024-07-28T19:18:39.240 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2024-07-28T19:18:39.240 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-07-28T19:18:39.294 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2024-07-28T19:18:39.294 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2024-07-28T19:18:39.294 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000381969 s, 1.3 MB/s 2024-07-28T19:18:39.295 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-07-28T19:18:39.346 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_2 2024-07-28T19:18:39.391 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-07-28T19:18:39.392 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-28T19:18:39.392 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 5h/5d Inode: 902 Links: 1 2024-07-28T19:18:39.392 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-28T19:18:39.392 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2024-07-28 19:09:14.300985823 +0000 2024-07-28T19:18:39.392 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2024-07-28 19:09:14.036988181 +0000 2024-07-28T19:18:39.392 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2024-07-28 19:09:14.036988181 +0000 2024-07-28T19:18:39.392 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2024-07-28T19:18:39.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-07-28T19:18:39.448 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2024-07-28T19:18:39.448 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2024-07-28T19:18:39.448 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000258831 s, 2.0 MB/s 2024-07-28T19:18:39.449 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-07-28T19:18:39.496 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_3 2024-07-28T19:18:39.543 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-07-28T19:18:39.544 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-28T19:18:39.544 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 5h/5d Inode: 924 Links: 1 2024-07-28T19:18:39.544 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-28T19:18:39.544 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2024-07-28 19:09:14.800981356 +0000 2024-07-28T19:18:39.544 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2024-07-28 19:09:14.528983785 +0000 2024-07-28T19:18:39.544 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2024-07-28 19:09:14.528983785 +0000 2024-07-28T19:18:39.544 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2024-07-28T19:18:39.544 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-07-28T19:18:39.596 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2024-07-28T19:18:39.596 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2024-07-28T19:18:39.596 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000303292 s, 1.7 MB/s 2024-07-28T19:18:39.597 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-07-28T19:18:39.644 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_4 2024-07-28T19:18:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-07-28T19:18:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-07-28T19:18:39.691 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 5h/5d Inode: 941 Links: 1 2024-07-28T19:18:39.691 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-07-28T19:18:39.691 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2024-07-28 19:09:15.276977105 +0000 2024-07-28T19:18:39.691 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2024-07-28 19:09:15.040979213 +0000 2024-07-28T19:18:39.692 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2024-07-28 19:09:15.040979213 +0000 2024-07-28T19:18:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2024-07-28T19:18:39.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-07-28T19:18:39.746 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2024-07-28T19:18:39.746 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2024-07-28T19:18:39.746 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000307227 s, 1.7 MB/s 2024-07-28T19:18:39.748 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-07-28T19:18:39.796 INFO:tasks.cephadm:Deploying osd.0 on smithi100 with /dev/vg_nvme/lv_4... 2024-07-28T19:18:39.796 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-07-28T19:18:40.480 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:40 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:40.481 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:40 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:40.481 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:40 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:40.481 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:40 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:40.809 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:40 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:40.809 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:40 smithi138 systemd[1]: Started Ceph mgr.x for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:18:40.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:40 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:18:40.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:40 smithi138 bash[23952]: cluster 2024-07-28T19:18:39.312896+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:40.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:40 smithi138 bash[23952]: audit 2024-07-28T19:18:40.618228+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:40.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:40 smithi138 bash[23952]: audit 2024-07-28T19:18:40.625582+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:40.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:40 smithi138 bash[23952]: audit 2024-07-28T19:18:40.632820+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:40.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:40 smithi138 bash[23952]: audit 2024-07-28T19:18:40.639886+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:41.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[21828]: cluster 2024-07-28T19:18:39.312896+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:41.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[21828]: audit 2024-07-28T19:18:40.618228+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:41.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[21828]: audit 2024-07-28T19:18:40.625582+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:41.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[21828]: audit 2024-07-28T19:18:40.632820+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:41.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[21828]: audit 2024-07-28T19:18:40.639886+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:41.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[31715]: cluster 2024-07-28T19:18:39.312896+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:41.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[31715]: audit 2024-07-28T19:18:40.618228+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:41.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[31715]: audit 2024-07-28T19:18:40.625582+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:41.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[31715]: audit 2024-07-28T19:18:40.632820+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:41.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:40 smithi100 bash[31715]: audit 2024-07-28T19:18:40.639886+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:42.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:41 smithi100 bash[21828]: audit 2024-07-28T19:18:40.700847+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:42.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:41 smithi100 bash[21828]: audit 2024-07-28T19:18:41.387320+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:42.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:41 smithi100 bash[31715]: audit 2024-07-28T19:18:40.700847+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:42.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:41 smithi100 bash[31715]: audit 2024-07-28T19:18:41.387320+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:42.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:41 smithi138 bash[23952]: audit 2024-07-28T19:18:40.700847+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:42.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:41 smithi138 bash[23952]: audit 2024-07-28T19:18:41.387320+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:42.059 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:41 smithi138 bash[24937]: debug 2024-07-28T19:18:41.697+0000 7fc230559640 1 -- 172.21.15.138:0/4121493177 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5588872ec9c0 con 0x5588872f8800 2024-07-28T19:18:42.059 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:41 smithi138 bash[24937]: debug 2024-07-28T19:18:41.805+0000 7fc232dc1ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:18:42.983 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:42 smithi138 bash[23952]: cluster 2024-07-28T19:18:41.313315+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:43.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:42 smithi100 bash[21828]: cluster 2024-07-28T19:18:41.313315+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:43.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:42 smithi100 bash[31715]: cluster 2024-07-28T19:18:41.313315+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:43.309 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:42 smithi138 bash[24937]: debug 2024-07-28T19:18:42.977+0000 7fc232dc1ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:18:43.309 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:43 smithi138 bash[24937]: debug 2024-07-28T19:18:43.041+0000 7fc232dc1ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:18:43.309 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:43 smithi138 bash[24937]: debug 2024-07-28T19:18:43.189+0000 7fc232dc1ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:18:44.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:43 smithi138 bash[24937]: debug 2024-07-28T19:18:43.813+0000 7fc232dc1ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:18:44.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:43 smithi138 bash[24937]: debug 2024-07-28T19:18:43.881+0000 7fc232dc1ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:18:44.107 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:43 smithi138 bash[24937]: debug 2024-07-28T19:18:43.949+0000 7fc232dc1ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:18:44.471 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:44 smithi138 bash[24937]: debug 2024-07-28T19:18:44.101+0000 7fc232dc1ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:18:44.471 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:44 smithi138 bash[24937]: debug 2024-07-28T19:18:44.177+0000 7fc232dc1ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:18:44.471 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:44 smithi138 bash[24937]: debug 2024-07-28T19:18:44.241+0000 7fc232dc1ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:18:44.520 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:18:44.774 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:44 smithi138 bash[23952]: cluster 2024-07-28T19:18:43.313669+0000 mgr.y (mgr.14150) 81 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:44.774 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:44 smithi138 bash[24937]: debug 2024-07-28T19:18:44.465+0000 7fc232dc1ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:18:44.774 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:44 smithi138 bash[24937]: debug 2024-07-28T19:18:44.529+0000 7fc232dc1ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:18:44.774 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:44 smithi138 bash[24937]: debug 2024-07-28T19:18:44.697+0000 7fc232dc1ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:18:45.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:44 smithi100 bash[21828]: cluster 2024-07-28T19:18:43.313669+0000 mgr.y (mgr.14150) 81 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:45.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:44 smithi100 bash[31715]: cluster 2024-07-28T19:18:43.313669+0000 mgr.y (mgr.14150) 81 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:45.059 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:44 smithi138 bash[24937]: debug 2024-07-28T19:18:44.769+0000 7fc232dc1ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:18:45.059 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:45 smithi138 bash[24937]: debug 2024-07-28T19:18:45.001+0000 7fc232dc1ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:18:45.559 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:45 smithi138 bash[24937]: debug 2024-07-28T19:18:45.157+0000 7fc232dc1ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:18:45.559 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:45 smithi138 bash[24937]: debug 2024-07-28T19:18:45.233+0000 7fc232dc1ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:18:45.559 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:45 smithi138 bash[24937]: debug 2024-07-28T19:18:45.381+0000 7fc232dc1ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:18:46.427 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[24937]: debug 2024-07-28T19:18:46.153+0000 7fc232dc1ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:18:46.427 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[24937]: debug 2024-07-28T19:18:46.229+0000 7fc232dc1ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:18:46.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: cluster 2024-07-28T19:18:45.314035+0000 mgr.y (mgr.14150) 82 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:46.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: audit 2024-07-28T19:18:45.640681+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: audit 2024-07-28T19:18:45.644606+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: audit 2024-07-28T19:18:46.027072+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: audit 2024-07-28T19:18:46.028474+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: audit 2024-07-28T19:18:46.036752+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: cephadm 2024-07-28T19:18:46.076084+0000 mgr.y (mgr.14150) 83 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: audit 2024-07-28T19:18:46.076689+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: audit 2024-07-28T19:18:46.078428+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: audit 2024-07-28T19:18:46.079679+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[21828]: cephadm 2024-07-28T19:18:46.081358+0000 mgr.y (mgr.14150) 84 : cephadm [INF] Reconfiguring daemon mgr.y on smithi100 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: cluster 2024-07-28T19:18:45.314035+0000 mgr.y (mgr.14150) 82 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: audit 2024-07-28T19:18:45.640681+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: audit 2024-07-28T19:18:45.644606+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: audit 2024-07-28T19:18:46.027072+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:46.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: audit 2024-07-28T19:18:46.028474+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:46.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: audit 2024-07-28T19:18:46.036752+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: cephadm 2024-07-28T19:18:46.076084+0000 mgr.y (mgr.14150) 83 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-07-28T19:18:46.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: audit 2024-07-28T19:18:46.076689+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:18:46.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: audit 2024-07-28T19:18:46.078428+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:18:46.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: audit 2024-07-28T19:18:46.079679+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:46.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:46 smithi100 bash[31715]: cephadm 2024-07-28T19:18:46.081358+0000 mgr.y (mgr.14150) 84 : cephadm [INF] Reconfiguring daemon mgr.y on smithi100 2024-07-28T19:18:46.808 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[24937]: debug 2024-07-28T19:18:46.421+0000 7fc232dc1ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[24937]: debug 2024-07-28T19:18:46.493+0000 7fc232dc1ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: cluster 2024-07-28T19:18:45.314035+0000 mgr.y (mgr.14150) 82 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: audit 2024-07-28T19:18:45.640681+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: audit 2024-07-28T19:18:45.644606+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: audit 2024-07-28T19:18:46.027072+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: audit 2024-07-28T19:18:46.028474+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: audit 2024-07-28T19:18:46.036752+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: cephadm 2024-07-28T19:18:46.076084+0000 mgr.y (mgr.14150) 83 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-07-28T19:18:46.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: audit 2024-07-28T19:18:46.076689+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:18:46.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: audit 2024-07-28T19:18:46.078428+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:18:46.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: audit 2024-07-28T19:18:46.079679+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:46.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[23952]: cephadm 2024-07-28T19:18:46.081358+0000 mgr.y (mgr.14150) 84 : cephadm [INF] Reconfiguring daemon mgr.y on smithi100 2024-07-28T19:18:47.309 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[24937]: debug 2024-07-28T19:18:46.805+0000 7fc232dc1ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:18:47.309 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:18:46 smithi138 bash[24937]: debug 2024-07-28T19:18:46.881+0000 7fc232dc1ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:18:48.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[21828]: cluster 2024-07-28T19:18:46.907393+0000 mon.a (mon.0) 236 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:18:48.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[21828]: audit 2024-07-28T19:18:46.910027+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:18:48.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[21828]: audit 2024-07-28T19:18:46.910829+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:18:48.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[21828]: audit 2024-07-28T19:18:46.911943+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:18:48.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[21828]: audit 2024-07-28T19:18:46.912472+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:18:48.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[31715]: cluster 2024-07-28T19:18:46.907393+0000 mon.a (mon.0) 236 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:18:48.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[31715]: audit 2024-07-28T19:18:46.910027+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:18:48.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[31715]: audit 2024-07-28T19:18:46.910829+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:18:48.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[31715]: audit 2024-07-28T19:18:46.911943+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:18:48.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:47 smithi100 bash[31715]: audit 2024-07-28T19:18:46.912472+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:18:48.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:47 smithi138 bash[23952]: cluster 2024-07-28T19:18:46.907393+0000 mon.a (mon.0) 236 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:18:48.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:47 smithi138 bash[23952]: audit 2024-07-28T19:18:46.910027+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:18:48.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:47 smithi138 bash[23952]: audit 2024-07-28T19:18:46.910829+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:18:48.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:47 smithi138 bash[23952]: audit 2024-07-28T19:18:46.911943+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:18:48.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:47 smithi138 bash[23952]: audit 2024-07-28T19:18:46.912472+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.138:0/1283414132' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:18:48.572 INFO:teuthology.orchestra.run.smithi100.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-07-28T19:18:48.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:48 smithi100 bash[21828]: cluster 2024-07-28T19:18:47.314460+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:48.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:48 smithi100 bash[21828]: cluster 2024-07-28T19:18:47.663927+0000 mon.a (mon.0) 237 : cluster [DBG] mgrmap e14: y(active, since 108s), standbys: x 2024-07-28T19:18:48.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:48 smithi100 bash[21828]: audit 2024-07-28T19:18:47.664226+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:18:48.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:48 smithi100 bash[31715]: cluster 2024-07-28T19:18:47.314460+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:48.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:48 smithi100 bash[31715]: cluster 2024-07-28T19:18:47.663927+0000 mon.a (mon.0) 237 : cluster [DBG] mgrmap e14: y(active, since 108s), standbys: x 2024-07-28T19:18:48.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:48 smithi100 bash[31715]: audit 2024-07-28T19:18:47.664226+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:18:48.751 INFO:teuthology.orchestra.run.smithi100.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-07-28T19:18:48.766 INFO:teuthology.orchestra.run.smithi100.stderr: stderr: 10+0 records in 2024-07-28T19:18:48.766 INFO:teuthology.orchestra.run.smithi100.stderr:10+0 records out 2024-07-28T19:18:48.766 INFO:teuthology.orchestra.run.smithi100.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133429 s, 786 MB/s 2024-07-28T19:18:48.766 INFO:teuthology.orchestra.run.smithi100.stderr:--> Zapping successful for: 2024-07-28T19:18:49.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:48 smithi138 bash[23952]: cluster 2024-07-28T19:18:47.314460+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:49.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:48 smithi138 bash[23952]: cluster 2024-07-28T19:18:47.663927+0000 mon.a (mon.0) 237 : cluster [DBG] mgrmap e14: y(active, since 108s), standbys: x 2024-07-28T19:18:49.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:48 smithi138 bash[23952]: audit 2024-07-28T19:18:47.664226+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:18:49.406 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch daemon add osd smithi100:vg_nvme/lv_4 2024-07-28T19:18:50.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[21828]: audit 2024-07-28T19:18:48.863936+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[21828]: audit 2024-07-28T19:18:48.868938+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[21828]: audit 2024-07-28T19:18:48.870230+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[21828]: audit 2024-07-28T19:18:48.871651+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[21828]: audit 2024-07-28T19:18:48.872347+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[21828]: audit 2024-07-28T19:18:48.877777+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[31715]: audit 2024-07-28T19:18:48.863936+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[31715]: audit 2024-07-28T19:18:48.868938+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[31715]: audit 2024-07-28T19:18:48.870230+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[31715]: audit 2024-07-28T19:18:48.871651+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:50.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[31715]: audit 2024-07-28T19:18:48.872347+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:50.252 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:49 smithi100 bash[31715]: audit 2024-07-28T19:18:48.877777+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:50.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:49 smithi138 bash[23952]: audit 2024-07-28T19:18:48.863936+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:50.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:49 smithi138 bash[23952]: audit 2024-07-28T19:18:48.868938+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:50.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:49 smithi138 bash[23952]: audit 2024-07-28T19:18:48.870230+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:18:50.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:49 smithi138 bash[23952]: audit 2024-07-28T19:18:48.871651+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:50.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:49 smithi138 bash[23952]: audit 2024-07-28T19:18:48.872347+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:18:50.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:49 smithi138 bash[23952]: audit 2024-07-28T19:18:48.877777+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:18:51.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:50 smithi100 bash[21828]: cluster 2024-07-28T19:18:49.314946+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:51.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:50 smithi100 bash[31715]: cluster 2024-07-28T19:18:49.314946+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:51.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:50 smithi138 bash[23952]: cluster 2024-07-28T19:18:49.314946+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:53.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:52 smithi100 bash[21828]: cluster 2024-07-28T19:18:51.315460+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:53.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:52 smithi100 bash[31715]: cluster 2024-07-28T19:18:51.315460+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:53.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:52 smithi138 bash[23952]: cluster 2024-07-28T19:18:51.315460+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:54.131 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:18:55.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:54 smithi100 bash[21828]: cluster 2024-07-28T19:18:53.315959+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:55.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:54 smithi100 bash[31715]: cluster 2024-07-28T19:18:53.315959+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:55.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:54 smithi138 bash[23952]: cluster 2024-07-28T19:18:53.315959+0000 mgr.y (mgr.14150) 88 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:57.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:56 smithi100 bash[21828]: cluster 2024-07-28T19:18:55.316289+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:57.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:56 smithi100 bash[31715]: cluster 2024-07-28T19:18:55.316289+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:57.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:56 smithi138 bash[23952]: cluster 2024-07-28T19:18:55.316289+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:58.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:57 smithi100 bash[21828]: audit 2024-07-28T19:18:57.411061+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:18:58.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:57 smithi100 bash[21828]: audit 2024-07-28T19:18:57.416211+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:18:58.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:57 smithi100 bash[21828]: audit 2024-07-28T19:18:57.417397+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:58.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:57 smithi100 bash[31715]: audit 2024-07-28T19:18:57.411061+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:18:58.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:57 smithi100 bash[31715]: audit 2024-07-28T19:18:57.416211+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:18:58.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:57 smithi100 bash[31715]: audit 2024-07-28T19:18:57.417397+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:58.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:57 smithi138 bash[23952]: audit 2024-07-28T19:18:57.411061+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:18:58.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:57 smithi138 bash[23952]: audit 2024-07-28T19:18:57.416211+0000 mon.a (mon.0) 246 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:18:58.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:57 smithi138 bash[23952]: audit 2024-07-28T19:18:57.417397+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:18:59.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:58 smithi100 bash[21828]: cluster 2024-07-28T19:18:57.316573+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:59.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:18:58 smithi100 bash[21828]: audit 2024-07-28T19:18:57.404624+0000 mgr.y (mgr.14150) 91 : audit [DBG] from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:59.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:58 smithi100 bash[31715]: cluster 2024-07-28T19:18:57.316573+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:59.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:18:58 smithi100 bash[31715]: audit 2024-07-28T19:18:57.404624+0000 mgr.y (mgr.14150) 91 : audit [DBG] from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:18:59.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:58 smithi138 bash[23952]: cluster 2024-07-28T19:18:57.316573+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:18:59.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:18:58 smithi138 bash[23952]: audit 2024-07-28T19:18:57.404624+0000 mgr.y (mgr.14150) 91 : audit [DBG] from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:19:01.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:00 smithi100 bash[21828]: cluster 2024-07-28T19:18:59.317097+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:01.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:00 smithi100 bash[31715]: cluster 2024-07-28T19:18:59.317097+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:01.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:00 smithi138 bash[23952]: cluster 2024-07-28T19:18:59.317097+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:03.228 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:02 smithi100 bash[21828]: cluster 2024-07-28T19:19:01.317493+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:03.229 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:02 smithi100 bash[31715]: cluster 2024-07-28T19:19:01.317493+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:03.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:02 smithi138 bash[23952]: cluster 2024-07-28T19:19:01.317493+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:05.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:04 smithi100 bash[21828]: cluster 2024-07-28T19:19:03.317862+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:05.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:04 smithi100 bash[31715]: cluster 2024-07-28T19:19:03.317862+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:05.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:04 smithi138 bash[23952]: cluster 2024-07-28T19:19:03.317862+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:06.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[21828]: audit 2024-07-28T19:19:05.609016+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.100:0/2090067099' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]: dispatch 2024-07-28T19:19:06.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[21828]: audit 2024-07-28T19:19:05.609628+0000 mon.a (mon.0) 248 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]: dispatch 2024-07-28T19:19:06.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[21828]: audit 2024-07-28T19:19:05.616326+0000 mon.a (mon.0) 249 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]': finished 2024-07-28T19:19:06.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[21828]: cluster 2024-07-28T19:19:05.616434+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-07-28T19:19:06.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[21828]: audit 2024-07-28T19:19:05.616631+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:06.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[31715]: audit 2024-07-28T19:19:05.609016+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.100:0/2090067099' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]: dispatch 2024-07-28T19:19:06.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[31715]: audit 2024-07-28T19:19:05.609628+0000 mon.a (mon.0) 248 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]: dispatch 2024-07-28T19:19:06.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[31715]: audit 2024-07-28T19:19:05.616326+0000 mon.a (mon.0) 249 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]': finished 2024-07-28T19:19:06.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[31715]: cluster 2024-07-28T19:19:05.616434+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-07-28T19:19:06.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:05 smithi100 bash[31715]: audit 2024-07-28T19:19:05.616631+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:06.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:05 smithi138 bash[23952]: audit 2024-07-28T19:19:05.609016+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.100:0/2090067099' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]: dispatch 2024-07-28T19:19:06.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:05 smithi138 bash[23952]: audit 2024-07-28T19:19:05.609628+0000 mon.a (mon.0) 248 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]: dispatch 2024-07-28T19:19:06.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:05 smithi138 bash[23952]: audit 2024-07-28T19:19:05.616326+0000 mon.a (mon.0) 249 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c4a49140-8022-42e2-bba9-333eb0e88545"}]': finished 2024-07-28T19:19:06.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:05 smithi138 bash[23952]: cluster 2024-07-28T19:19:05.616434+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-07-28T19:19:06.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:05 smithi138 bash[23952]: audit 2024-07-28T19:19:05.616631+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:07.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:06 smithi100 bash[21828]: cluster 2024-07-28T19:19:05.318225+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:07.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:06 smithi100 bash[21828]: audit 2024-07-28T19:19:06.236606+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.100:0/2250503463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:19:07.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:06 smithi100 bash[31715]: cluster 2024-07-28T19:19:05.318225+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:07.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:06 smithi100 bash[31715]: audit 2024-07-28T19:19:06.236606+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.100:0/2250503463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:19:07.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:06 smithi138 bash[23952]: cluster 2024-07-28T19:19:05.318225+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:07.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:06 smithi138 bash[23952]: audit 2024-07-28T19:19:06.236606+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.100:0/2250503463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:19:09.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:08 smithi100 bash[21828]: cluster 2024-07-28T19:19:07.318653+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:09.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:08 smithi100 bash[31715]: cluster 2024-07-28T19:19:07.318653+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:09.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:08 smithi138 bash[23952]: cluster 2024-07-28T19:19:07.318653+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:11.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:10 smithi100 bash[21828]: cluster 2024-07-28T19:19:09.319056+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:11.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:10 smithi100 bash[31715]: cluster 2024-07-28T19:19:09.319056+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:11.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:10 smithi138 bash[23952]: cluster 2024-07-28T19:19:09.319056+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:13.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:12 smithi100 bash[21828]: cluster 2024-07-28T19:19:11.319494+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:13.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:12 smithi100 bash[31715]: cluster 2024-07-28T19:19:11.319494+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:13.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:12 smithi138 bash[23952]: cluster 2024-07-28T19:19:11.319494+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:15.217 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:14 smithi100 bash[21828]: cluster 2024-07-28T19:19:13.319895+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:15.218 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:14 smithi100 bash[31715]: cluster 2024-07-28T19:19:13.319895+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:15.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:14 smithi138 bash[23952]: cluster 2024-07-28T19:19:13.319895+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:17.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:16 smithi100 bash[21828]: cluster 2024-07-28T19:19:15.320228+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:17.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:16 smithi100 bash[31715]: cluster 2024-07-28T19:19:15.320228+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:17.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:16 smithi138 bash[23952]: cluster 2024-07-28T19:19:15.320228+0000 mgr.y (mgr.14150) 100 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:19.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:18 smithi100 bash[21828]: cluster 2024-07-28T19:19:17.320537+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:19.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:18 smithi100 bash[21828]: audit 2024-07-28T19:19:17.948159+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:19:19.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:18 smithi100 bash[21828]: audit 2024-07-28T19:19:17.949858+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:19.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:18 smithi100 bash[21828]: cephadm 2024-07-28T19:19:17.951086+0000 mgr.y (mgr.14150) 102 : cephadm [INF] Deploying daemon osd.0 on smithi100 2024-07-28T19:19:19.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:18 smithi100 bash[31715]: cluster 2024-07-28T19:19:17.320537+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:19.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:18 smithi100 bash[31715]: audit 2024-07-28T19:19:17.948159+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:19:19.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:18 smithi100 bash[31715]: audit 2024-07-28T19:19:17.949858+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:19.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:18 smithi100 bash[31715]: cephadm 2024-07-28T19:19:17.951086+0000 mgr.y (mgr.14150) 102 : cephadm [INF] Deploying daemon osd.0 on smithi100 2024-07-28T19:19:19.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:18 smithi138 bash[23952]: cluster 2024-07-28T19:19:17.320537+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:19.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:18 smithi138 bash[23952]: audit 2024-07-28T19:19:17.948159+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:19:19.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:18 smithi138 bash[23952]: audit 2024-07-28T19:19:17.949858+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:19.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:18 smithi138 bash[23952]: cephadm 2024-07-28T19:19:17.951086+0000 mgr.y (mgr.14150) 102 : cephadm [INF] Deploying daemon osd.0 on smithi100 2024-07-28T19:19:21.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:20 smithi100 bash[31715]: cluster 2024-07-28T19:19:19.320796+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:21.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:20 smithi100 bash[21828]: cluster 2024-07-28T19:19:19.320796+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:20 smithi138 bash[23952]: cluster 2024-07-28T19:19:19.320796+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:22.199 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:22 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:19:22.200 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:22 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:19:22.200 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:19:22 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:19:22.499 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:19:22 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:19:22.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:22 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:19:22.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:22 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:19:23.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:22 smithi100 bash[21828]: cluster 2024-07-28T19:19:21.321135+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:23.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:22 smithi100 bash[21828]: audit 2024-07-28T19:19:22.462424+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:19:23.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:22 smithi100 bash[21828]: audit 2024-07-28T19:19:22.467380+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:23.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:22 smithi100 bash[21828]: audit 2024-07-28T19:19:22.472943+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:23.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:22 smithi100 bash[31715]: cluster 2024-07-28T19:19:21.321135+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:23.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:22 smithi100 bash[31715]: audit 2024-07-28T19:19:22.462424+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:19:23.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:22 smithi100 bash[31715]: audit 2024-07-28T19:19:22.467380+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:23.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:22 smithi100 bash[31715]: audit 2024-07-28T19:19:22.472943+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:23.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:22 smithi138 bash[23952]: cluster 2024-07-28T19:19:21.321135+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:23.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:22 smithi138 bash[23952]: audit 2024-07-28T19:19:22.462424+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:19:23.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:22 smithi138 bash[23952]: audit 2024-07-28T19:19:22.467380+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:23.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:22 smithi138 bash[23952]: audit 2024-07-28T19:19:22.472943+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:25.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:24 smithi100 bash[21828]: cluster 2024-07-28T19:19:23.321540+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:25.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:24 smithi100 bash[31715]: cluster 2024-07-28T19:19:23.321540+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:25.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:24 smithi138 bash[23952]: cluster 2024-07-28T19:19:23.321540+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:27.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:26 smithi100 bash[21828]: cluster 2024-07-28T19:19:25.321906+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:27.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:26 smithi100 bash[31715]: cluster 2024-07-28T19:19:25.321906+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:27.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:26 smithi138 bash[23952]: cluster 2024-07-28T19:19:25.321906+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:29.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:28 smithi100 bash[21828]: cluster 2024-07-28T19:19:27.322250+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:29.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:28 smithi100 bash[31715]: cluster 2024-07-28T19:19:27.322250+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:29.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:28 smithi138 bash[23952]: cluster 2024-07-28T19:19:27.322250+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:30.175 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[21828]: audit 2024-07-28T19:19:29.044764+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.175 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[21828]: audit 2024-07-28T19:19:29.048613+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.175 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[21828]: audit 2024-07-28T19:19:29.380913+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:30.175 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[21828]: audit 2024-07-28T19:19:29.381568+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:19:30.175 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[21828]: audit 2024-07-28T19:19:29.386252+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.175 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[21828]: audit 2024-07-28T19:19:29.878747+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:19:30.175 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[21828]: audit 2024-07-28T19:19:29.879147+0000 mon.a (mon.0) 262 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:19:30.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[31715]: audit 2024-07-28T19:19:29.044764+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.176 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[31715]: audit 2024-07-28T19:19:29.048613+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.176 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[31715]: audit 2024-07-28T19:19:29.380913+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:30.176 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[31715]: audit 2024-07-28T19:19:29.381568+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:19:30.176 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[31715]: audit 2024-07-28T19:19:29.386252+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.176 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[31715]: audit 2024-07-28T19:19:29.878747+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:19:30.176 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:30 smithi100 bash[31715]: audit 2024-07-28T19:19:29.879147+0000 mon.a (mon.0) 262 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:19:30.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:30 smithi138 bash[23952]: audit 2024-07-28T19:19:29.044764+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:30 smithi138 bash[23952]: audit 2024-07-28T19:19:29.048613+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:30 smithi138 bash[23952]: audit 2024-07-28T19:19:29.380913+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:30.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:30 smithi138 bash[23952]: audit 2024-07-28T19:19:29.381568+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:19:30.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:30 smithi138 bash[23952]: audit 2024-07-28T19:19:29.386252+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:30.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:30 smithi138 bash[23952]: audit 2024-07-28T19:19:29.878747+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:19:30.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:30 smithi138 bash[23952]: audit 2024-07-28T19:19:29.879147+0000 mon.a (mon.0) 262 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:19:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:31 smithi138 bash[23952]: cluster 2024-07-28T19:19:29.322542+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:31 smithi138 bash[23952]: audit 2024-07-28T19:19:30.391758+0000 mon.a (mon.0) 263 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-28T19:19:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:31 smithi138 bash[23952]: cluster 2024-07-28T19:19:30.391881+0000 mon.a (mon.0) 264 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-07-28T19:19:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:31 smithi138 bash[23952]: audit 2024-07-28T19:19:30.392082+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:31 smithi138 bash[23952]: audit 2024-07-28T19:19:30.393257+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:19:31.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:31 smithi138 bash[23952]: audit 2024-07-28T19:19:30.393989+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:19:31.402 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[21828]: cluster 2024-07-28T19:19:29.322542+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[21828]: audit 2024-07-28T19:19:30.391758+0000 mon.a (mon.0) 263 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[21828]: cluster 2024-07-28T19:19:30.391881+0000 mon.a (mon.0) 264 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[21828]: audit 2024-07-28T19:19:30.392082+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[21828]: audit 2024-07-28T19:19:30.393257+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[21828]: audit 2024-07-28T19:19:30.393989+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[31715]: cluster 2024-07-28T19:19:29.322542+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[31715]: audit 2024-07-28T19:19:30.391758+0000 mon.a (mon.0) 263 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[31715]: cluster 2024-07-28T19:19:30.391881+0000 mon.a (mon.0) 264 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-07-28T19:19:31.403 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[31715]: audit 2024-07-28T19:19:30.392082+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:31.404 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[31715]: audit 2024-07-28T19:19:30.393257+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:19:31.404 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:31 smithi100 bash[31715]: audit 2024-07-28T19:19:30.393989+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:19:32.647 INFO:teuthology.orchestra.run.smithi100.stdout:Created osd(s) 0 on host 'smithi100' 2024-07-28T19:19:32.664 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[21828]: cluster 2024-07-28T19:19:31.322906+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[21828]: audit 2024-07-28T19:19:31.395642+0000 mon.a (mon.0) 267 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[21828]: cluster 2024-07-28T19:19:31.395853+0000 mon.a (mon.0) 268 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[21828]: audit 2024-07-28T19:19:31.396000+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[21828]: audit 2024-07-28T19:19:31.403496+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[31715]: cluster 2024-07-28T19:19:31.322906+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[31715]: audit 2024-07-28T19:19:31.395642+0000 mon.a (mon.0) 267 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[31715]: cluster 2024-07-28T19:19:31.395853+0000 mon.a (mon.0) 268 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[31715]: audit 2024-07-28T19:19:31.396000+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:32.665 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:32 smithi100 bash[31715]: audit 2024-07-28T19:19:31.403496+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:32.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:32 smithi138 bash[23952]: cluster 2024-07-28T19:19:31.322906+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-07-28T19:19:32.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:32 smithi138 bash[23952]: audit 2024-07-28T19:19:31.395642+0000 mon.a (mon.0) 267 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:19:32.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:32 smithi138 bash[23952]: cluster 2024-07-28T19:19:31.395853+0000 mon.a (mon.0) 268 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-07-28T19:19:32.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:32 smithi138 bash[23952]: audit 2024-07-28T19:19:31.396000+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:32.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:32 smithi138 bash[23952]: audit 2024-07-28T19:19:31.403496+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:33.303 DEBUG:teuthology.orchestra.run.smithi100:osd.0> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.0.service 2024-07-28T19:19:33.306 INFO:tasks.cephadm:Deploying osd.1 on smithi100 with /dev/vg_nvme/lv_3... 2024-07-28T19:19:33.306 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-07-28T19:19:33.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[21828]: cluster 2024-07-28T19:19:30.915957+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:19:33.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[21828]: cluster 2024-07-28T19:19:30.916064+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[21828]: cluster 2024-07-28T19:19:32.398929+0000 mon.a (mon.0) 271 : cluster [INF] osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215] boot 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[21828]: cluster 2024-07-28T19:19:32.398998+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[21828]: audit 2024-07-28T19:19:32.399744+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[21828]: audit 2024-07-28T19:19:32.624281+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[21828]: audit 2024-07-28T19:19:32.632964+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[21828]: audit 2024-07-28T19:19:32.644256+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[31715]: cluster 2024-07-28T19:19:30.915957+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[31715]: cluster 2024-07-28T19:19:30.916064+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[31715]: cluster 2024-07-28T19:19:32.398929+0000 mon.a (mon.0) 271 : cluster [INF] osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215] boot 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[31715]: cluster 2024-07-28T19:19:32.398998+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[31715]: audit 2024-07-28T19:19:32.399744+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:33.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[31715]: audit 2024-07-28T19:19:32.624281+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:19:33.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[31715]: audit 2024-07-28T19:19:32.632964+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:33.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:33 smithi100 bash[31715]: audit 2024-07-28T19:19:32.644256+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:33 smithi138 bash[23952]: cluster 2024-07-28T19:19:30.915957+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:19:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:33 smithi138 bash[23952]: cluster 2024-07-28T19:19:30.916064+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:19:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:33 smithi138 bash[23952]: cluster 2024-07-28T19:19:32.398929+0000 mon.a (mon.0) 271 : cluster [INF] osd.0 [v2:172.21.15.100:6802/2528423215,v1:172.21.15.100:6803/2528423215] boot 2024-07-28T19:19:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:33 smithi138 bash[23952]: cluster 2024-07-28T19:19:32.398998+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-07-28T19:19:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:33 smithi138 bash[23952]: audit 2024-07-28T19:19:32.399744+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:19:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:33 smithi138 bash[23952]: audit 2024-07-28T19:19:32.624281+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:19:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:33 smithi138 bash[23952]: audit 2024-07-28T19:19:32.632964+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:33 smithi138 bash[23952]: audit 2024-07-28T19:19:32.644256+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:35.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:34 smithi100 bash[21828]: cluster 2024-07-28T19:19:33.323451+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:35.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:34 smithi100 bash[21828]: cluster 2024-07-28T19:19:33.655648+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-07-28T19:19:35.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:34 smithi100 bash[31715]: cluster 2024-07-28T19:19:33.323451+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:35.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:34 smithi100 bash[31715]: cluster 2024-07-28T19:19:33.655648+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-07-28T19:19:35.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:34 smithi138 bash[23952]: cluster 2024-07-28T19:19:33.323451+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:35.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:34 smithi138 bash[23952]: cluster 2024-07-28T19:19:33.655648+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-07-28T19:19:37.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:36 smithi100 bash[21828]: cluster 2024-07-28T19:19:35.323896+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:37.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:36 smithi100 bash[31715]: cluster 2024-07-28T19:19:35.323896+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:37.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:36 smithi138 bash[23952]: cluster 2024-07-28T19:19:35.323896+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:37.394 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:19:38.990 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:38 smithi100 bash[21828]: cluster 2024-07-28T19:19:37.324369+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:38.990 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:38 smithi100 bash[31715]: cluster 2024-07-28T19:19:37.324369+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:39.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:38 smithi138 bash[23952]: cluster 2024-07-28T19:19:37.324369+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:41.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:40 smithi100 bash[21828]: cluster 2024-07-28T19:19:39.324698+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:41.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:40 smithi100 bash[31715]: cluster 2024-07-28T19:19:39.324698+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:41.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:40 smithi138 bash[23952]: cluster 2024-07-28T19:19:39.324698+0000 mgr.y (mgr.14150) 113 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:42.064 INFO:teuthology.orchestra.run.smithi100.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-07-28T19:19:42.212 INFO:teuthology.orchestra.run.smithi100.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-07-28T19:19:42.228 INFO:teuthology.orchestra.run.smithi100.stderr: stderr: 10+0 records in 2024-07-28T19:19:42.228 INFO:teuthology.orchestra.run.smithi100.stderr:10+0 records out 2024-07-28T19:19:42.228 INFO:teuthology.orchestra.run.smithi100.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.013989 s, 750 MB/s 2024-07-28T19:19:42.228 INFO:teuthology.orchestra.run.smithi100.stderr:--> Zapping successful for: 2024-07-28T19:19:42.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:42 smithi100 bash[21828]: cluster 2024-07-28T19:19:41.325239+0000 mgr.y (mgr.14150) 114 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:42.899 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:42 smithi100 bash[31715]: cluster 2024-07-28T19:19:41.325239+0000 mgr.y (mgr.14150) 114 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:42.944 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch daemon add osd smithi100:vg_nvme/lv_3 2024-07-28T19:19:43.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:42 smithi138 bash[23952]: cluster 2024-07-28T19:19:41.325239+0000 mgr.y (mgr.14150) 114 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:44.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: cluster 2024-07-28T19:19:43.325738+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:44.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: cephadm 2024-07-28T19:19:43.353509+0000 mgr.y (mgr.14150) 116 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:19:44.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: audit 2024-07-28T19:19:43.361594+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: audit 2024-07-28T19:19:43.370254+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: audit 2024-07-28T19:19:43.372108+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: cephadm 2024-07-28T19:19:43.373454+0000 mgr.y (mgr.14150) 117 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 16241M 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: audit 2024-07-28T19:19:43.381755+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: audit 2024-07-28T19:19:43.383874+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: audit 2024-07-28T19:19:43.385474+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[31715]: audit 2024-07-28T19:19:43.396764+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: cluster 2024-07-28T19:19:43.325738+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: cephadm 2024-07-28T19:19:43.353509+0000 mgr.y (mgr.14150) 116 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: audit 2024-07-28T19:19:43.361594+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: audit 2024-07-28T19:19:43.370254+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: audit 2024-07-28T19:19:43.372108+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:19:44.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: cephadm 2024-07-28T19:19:43.373454+0000 mgr.y (mgr.14150) 117 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 16241M 2024-07-28T19:19:44.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: audit 2024-07-28T19:19:43.381755+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: audit 2024-07-28T19:19:43.383874+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:44.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: audit 2024-07-28T19:19:43.385474+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:19:44.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:44 smithi100 bash[21828]: audit 2024-07-28T19:19:43.396764+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: cluster 2024-07-28T19:19:43.325738+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: cephadm 2024-07-28T19:19:43.353509+0000 mgr.y (mgr.14150) 116 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: audit 2024-07-28T19:19:43.361594+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: audit 2024-07-28T19:19:43.370254+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: audit 2024-07-28T19:19:43.372108+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: cephadm 2024-07-28T19:19:43.373454+0000 mgr.y (mgr.14150) 117 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 16241M 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: audit 2024-07-28T19:19:43.381755+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: audit 2024-07-28T19:19:43.383874+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:44.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: audit 2024-07-28T19:19:43.385474+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:19:44.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:44 smithi138 bash[23952]: audit 2024-07-28T19:19:43.396764+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:19:46.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:46 smithi100 bash[21828]: cluster 2024-07-28T19:19:45.326231+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:46.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:46 smithi100 bash[31715]: cluster 2024-07-28T19:19:45.326231+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:46.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:46 smithi138 bash[23952]: cluster 2024-07-28T19:19:45.326231+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:47.668 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:19:48.662 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:48 smithi100 bash[21828]: cluster 2024-07-28T19:19:47.326679+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:48.663 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:48 smithi100 bash[31715]: cluster 2024-07-28T19:19:47.326679+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:48.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:48 smithi138 bash[23952]: cluster 2024-07-28T19:19:47.326679+0000 mgr.y (mgr.14150) 119 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:50 smithi100 bash[21828]: cluster 2024-07-28T19:19:49.327224+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:50 smithi100 bash[31715]: cluster 2024-07-28T19:19:49.327224+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:50.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:50 smithi138 bash[23952]: cluster 2024-07-28T19:19:49.327224+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:51.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:51 smithi100 bash[21828]: audit 2024-07-28T19:19:50.615333+0000 mgr.y (mgr.14150) 121 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:19:51.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:51 smithi100 bash[21828]: audit 2024-07-28T19:19:50.621436+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:19:51.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:51 smithi100 bash[21828]: audit 2024-07-28T19:19:50.626705+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:19:51.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:51 smithi100 bash[21828]: audit 2024-07-28T19:19:50.627904+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:51.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:51 smithi100 bash[31715]: audit 2024-07-28T19:19:50.615333+0000 mgr.y (mgr.14150) 121 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:19:51.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:51 smithi100 bash[31715]: audit 2024-07-28T19:19:50.621436+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:19:51.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:51 smithi100 bash[31715]: audit 2024-07-28T19:19:50.626705+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:19:51.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:51 smithi100 bash[31715]: audit 2024-07-28T19:19:50.627904+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:51.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:51 smithi138 bash[23952]: audit 2024-07-28T19:19:50.615333+0000 mgr.y (mgr.14150) 121 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:19:51.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:51 smithi138 bash[23952]: audit 2024-07-28T19:19:50.621436+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:19:51.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:51 smithi138 bash[23952]: audit 2024-07-28T19:19:50.626705+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:19:51.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:51 smithi138 bash[23952]: audit 2024-07-28T19:19:50.627904+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:19:52.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:52 smithi100 bash[21828]: cluster 2024-07-28T19:19:51.327698+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:52.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:52 smithi100 bash[31715]: cluster 2024-07-28T19:19:51.327698+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:52.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:52 smithi138 bash[23952]: cluster 2024-07-28T19:19:51.327698+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:54.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:54 smithi100 bash[21828]: cluster 2024-07-28T19:19:53.328242+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:54.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:54 smithi100 bash[31715]: cluster 2024-07-28T19:19:53.328242+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:54.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:54 smithi138 bash[23952]: cluster 2024-07-28T19:19:53.328242+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:56.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:56 smithi100 bash[21828]: cluster 2024-07-28T19:19:55.328806+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:56.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:56 smithi100 bash[31715]: cluster 2024-07-28T19:19:55.328806+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:56.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:56 smithi138 bash[23952]: cluster 2024-07-28T19:19:55.328806+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:58.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:58 smithi100 bash[31715]: cluster 2024-07-28T19:19:57.329300+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:58.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:58 smithi100 bash[21828]: cluster 2024-07-28T19:19:57.329300+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:58.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:58 smithi138 bash[23952]: cluster 2024-07-28T19:19:57.329300+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:19:59.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:59 smithi100 bash[21828]: audit 2024-07-28T19:19:58.926306+0000 mon.a (mon.0) 288 : audit [INF] from='client.? 172.21.15.100:0/2604047697' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4117fbb5-c88e-4d12-bc76-77050b2c0425"}]: dispatch 2024-07-28T19:19:59.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:59 smithi100 bash[21828]: audit 2024-07-28T19:19:58.932024+0000 mon.a (mon.0) 289 : audit [INF] from='client.? 172.21.15.100:0/2604047697' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4117fbb5-c88e-4d12-bc76-77050b2c0425"}]': finished 2024-07-28T19:19:59.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:59 smithi100 bash[21828]: cluster 2024-07-28T19:19:58.932107+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-07-28T19:19:59.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:19:59 smithi100 bash[21828]: audit 2024-07-28T19:19:58.932189+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:19:59.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:59 smithi100 bash[31715]: audit 2024-07-28T19:19:58.926306+0000 mon.a (mon.0) 288 : audit [INF] from='client.? 172.21.15.100:0/2604047697' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4117fbb5-c88e-4d12-bc76-77050b2c0425"}]: dispatch 2024-07-28T19:19:59.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:59 smithi100 bash[31715]: audit 2024-07-28T19:19:58.932024+0000 mon.a (mon.0) 289 : audit [INF] from='client.? 172.21.15.100:0/2604047697' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4117fbb5-c88e-4d12-bc76-77050b2c0425"}]': finished 2024-07-28T19:19:59.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:59 smithi100 bash[31715]: cluster 2024-07-28T19:19:58.932107+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-07-28T19:19:59.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:19:59 smithi100 bash[31715]: audit 2024-07-28T19:19:58.932189+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:19:59.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:59 smithi138 bash[23952]: audit 2024-07-28T19:19:58.926306+0000 mon.a (mon.0) 288 : audit [INF] from='client.? 172.21.15.100:0/2604047697' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4117fbb5-c88e-4d12-bc76-77050b2c0425"}]: dispatch 2024-07-28T19:19:59.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:59 smithi138 bash[23952]: audit 2024-07-28T19:19:58.932024+0000 mon.a (mon.0) 289 : audit [INF] from='client.? 172.21.15.100:0/2604047697' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4117fbb5-c88e-4d12-bc76-77050b2c0425"}]': finished 2024-07-28T19:19:59.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:59 smithi138 bash[23952]: cluster 2024-07-28T19:19:58.932107+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-07-28T19:19:59.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:19:59 smithi138 bash[23952]: audit 2024-07-28T19:19:58.932189+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:00.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:00 smithi100 bash[31715]: cluster 2024-07-28T19:19:59.329592+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:00.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:00 smithi100 bash[31715]: audit 2024-07-28T19:19:59.551404+0000 mon.a (mon.0) 292 : audit [DBG] from='client.? 172.21.15.100:0/1759412943' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:20:00.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:00 smithi100 bash[31715]: cluster 2024-07-28T19:20:00.000115+0000 mon.a (mon.0) 293 : cluster [INF] overall HEALTH_OK 2024-07-28T19:20:00.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:00 smithi100 bash[21828]: cluster 2024-07-28T19:19:59.329592+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:00.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:00 smithi100 bash[21828]: audit 2024-07-28T19:19:59.551404+0000 mon.a (mon.0) 292 : audit [DBG] from='client.? 172.21.15.100:0/1759412943' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:20:00.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:00 smithi100 bash[21828]: cluster 2024-07-28T19:20:00.000115+0000 mon.a (mon.0) 293 : cluster [INF] overall HEALTH_OK 2024-07-28T19:20:00.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:00 smithi138 bash[23952]: cluster 2024-07-28T19:19:59.329592+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:00.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:00 smithi138 bash[23952]: audit 2024-07-28T19:19:59.551404+0000 mon.a (mon.0) 292 : audit [DBG] from='client.? 172.21.15.100:0/1759412943' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:20:00.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:00 smithi138 bash[23952]: cluster 2024-07-28T19:20:00.000115+0000 mon.a (mon.0) 293 : cluster [INF] overall HEALTH_OK 2024-07-28T19:20:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:02 smithi100 bash[31715]: cluster 2024-07-28T19:20:01.329982+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:02.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:02 smithi100 bash[21828]: cluster 2024-07-28T19:20:01.329982+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:02.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:02 smithi138 bash[23952]: cluster 2024-07-28T19:20:01.329982+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:04.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:04 smithi100 bash[31715]: cluster 2024-07-28T19:20:03.330412+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:04.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:04 smithi100 bash[21828]: cluster 2024-07-28T19:20:03.330412+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:04.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:04 smithi138 bash[23952]: cluster 2024-07-28T19:20:03.330412+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:06.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:06 smithi100 bash[21828]: cluster 2024-07-28T19:20:05.330845+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:06.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:06 smithi100 bash[31715]: cluster 2024-07-28T19:20:05.330845+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:06.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:06 smithi138 bash[23952]: cluster 2024-07-28T19:20:05.330845+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:08.665 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:08 smithi100 bash[21828]: cluster 2024-07-28T19:20:07.331325+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:08.666 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:08 smithi100 bash[31715]: cluster 2024-07-28T19:20:07.331325+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:08.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:08 smithi138 bash[23952]: cluster 2024-07-28T19:20:07.331325+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:10.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:10 smithi100 bash[21828]: cluster 2024-07-28T19:20:09.331639+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:10.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:10 smithi100 bash[31715]: cluster 2024-07-28T19:20:09.331639+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:10.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:10 smithi138 bash[23952]: cluster 2024-07-28T19:20:09.331639+0000 mgr.y (mgr.14150) 131 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:11.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:11 smithi100 bash[21828]: audit 2024-07-28T19:20:11.205318+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:20:11.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:11 smithi100 bash[21828]: audit 2024-07-28T19:20:11.207252+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:11.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:11 smithi100 bash[31715]: audit 2024-07-28T19:20:11.205318+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:20:11.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:11 smithi100 bash[31715]: audit 2024-07-28T19:20:11.207252+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:11.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:11 smithi138 bash[23952]: audit 2024-07-28T19:20:11.205318+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:20:11.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:11 smithi138 bash[23952]: audit 2024-07-28T19:20:11.207252+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:12 smithi100 bash[21828]: cephadm 2024-07-28T19:20:11.208777+0000 mgr.y (mgr.14150) 132 : cephadm [INF] Deploying daemon osd.1 on smithi100 2024-07-28T19:20:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:12 smithi100 bash[21828]: cluster 2024-07-28T19:20:11.332026+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:12.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:12 smithi100 bash[31715]: cephadm 2024-07-28T19:20:11.208777+0000 mgr.y (mgr.14150) 132 : cephadm [INF] Deploying daemon osd.1 on smithi100 2024-07-28T19:20:12.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:12 smithi100 bash[31715]: cluster 2024-07-28T19:20:11.332026+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:12 smithi138 bash[23952]: cephadm 2024-07-28T19:20:11.208777+0000 mgr.y (mgr.14150) 132 : cephadm [INF] Deploying daemon osd.1 on smithi100 2024-07-28T19:20:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:12 smithi138 bash[23952]: cluster 2024-07-28T19:20:11.332026+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:14.469 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:14 smithi100 bash[21828]: cluster 2024-07-28T19:20:13.332505+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:14.469 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:14 smithi100 bash[31715]: cluster 2024-07-28T19:20:13.332505+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:14.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:14 smithi138 bash[23952]: cluster 2024-07-28T19:20:13.332505+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:15.396 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:20:15.396 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:20:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:20:15.397 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:20:15.397 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:20:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:20:15.725 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:20:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:20:15.725 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:20:15.726 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:20:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:20:15.726 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:20:16.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:16 smithi100 bash[21828]: cluster 2024-07-28T19:20:15.332821+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:16.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:16 smithi100 bash[21828]: audit 2024-07-28T19:20:15.651831+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:20:16.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:16 smithi100 bash[21828]: audit 2024-07-28T19:20:15.655811+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:16.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:16 smithi100 bash[21828]: audit 2024-07-28T19:20:15.661560+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:16.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:16 smithi100 bash[31715]: cluster 2024-07-28T19:20:15.332821+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:16.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:16 smithi100 bash[31715]: audit 2024-07-28T19:20:15.651831+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:20:16.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:16 smithi100 bash[31715]: audit 2024-07-28T19:20:15.655811+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:16.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:16 smithi100 bash[31715]: audit 2024-07-28T19:20:15.661560+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:16 smithi138 bash[23952]: cluster 2024-07-28T19:20:15.332821+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:16 smithi138 bash[23952]: audit 2024-07-28T19:20:15.651831+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:20:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:16 smithi138 bash[23952]: audit 2024-07-28T19:20:15.655811+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:16 smithi138 bash[23952]: audit 2024-07-28T19:20:15.661560+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:18.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:18 smithi100 bash[21828]: cluster 2024-07-28T19:20:17.333215+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:18.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:18 smithi100 bash[31715]: cluster 2024-07-28T19:20:17.333215+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:18.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:18 smithi138 bash[23952]: cluster 2024-07-28T19:20:17.333215+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:20.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:20 smithi100 bash[21828]: cluster 2024-07-28T19:20:19.333647+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:20.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:20 smithi100 bash[31715]: cluster 2024-07-28T19:20:19.333647+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:20.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:20 smithi138 bash[23952]: cluster 2024-07-28T19:20:19.333647+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:23.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[31715]: cluster 2024-07-28T19:20:21.334102+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:23.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[31715]: audit 2024-07-28T19:20:21.696726+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[31715]: audit 2024-07-28T19:20:21.703300+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[31715]: audit 2024-07-28T19:20:21.704063+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:23.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[31715]: audit 2024-07-28T19:20:21.704715+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:20:23.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[31715]: audit 2024-07-28T19:20:21.713570+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[31715]: audit 2024-07-28T19:20:22.330181+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:20:23.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[31715]: audit 2024-07-28T19:20:22.330525+0000 mon.a (mon.0) 304 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:20:23.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[21828]: cluster 2024-07-28T19:20:21.334102+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:23.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[21828]: audit 2024-07-28T19:20:21.696726+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[21828]: audit 2024-07-28T19:20:21.703300+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[21828]: audit 2024-07-28T19:20:21.704063+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:23.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[21828]: audit 2024-07-28T19:20:21.704715+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:20:23.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[21828]: audit 2024-07-28T19:20:21.713570+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[21828]: audit 2024-07-28T19:20:22.330181+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:20:23.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:22 smithi100 bash[21828]: audit 2024-07-28T19:20:22.330525+0000 mon.a (mon.0) 304 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:20:23.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:22 smithi138 bash[23952]: cluster 2024-07-28T19:20:21.334102+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:23.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:22 smithi138 bash[23952]: audit 2024-07-28T19:20:21.696726+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:22 smithi138 bash[23952]: audit 2024-07-28T19:20:21.703300+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:22 smithi138 bash[23952]: audit 2024-07-28T19:20:21.704063+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:23.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:22 smithi138 bash[23952]: audit 2024-07-28T19:20:21.704715+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:20:23.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:22 smithi138 bash[23952]: audit 2024-07-28T19:20:21.713570+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:23.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:22 smithi138 bash[23952]: audit 2024-07-28T19:20:22.330181+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:20:23.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:22 smithi138 bash[23952]: audit 2024-07-28T19:20:22.330525+0000 mon.a (mon.0) 304 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:20:24.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[21828]: audit 2024-07-28T19:20:22.714927+0000 mon.a (mon.0) 305 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-28T19:20:24.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[21828]: cluster 2024-07-28T19:20:22.714994+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-07-28T19:20:24.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[21828]: audit 2024-07-28T19:20:22.715237+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:24.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[21828]: audit 2024-07-28T19:20:22.718575+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:20:24.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[21828]: audit 2024-07-28T19:20:22.719224+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:20:24.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[31715]: audit 2024-07-28T19:20:22.714927+0000 mon.a (mon.0) 305 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-28T19:20:24.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[31715]: cluster 2024-07-28T19:20:22.714994+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-07-28T19:20:24.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[31715]: audit 2024-07-28T19:20:22.715237+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:24.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[31715]: audit 2024-07-28T19:20:22.718575+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:20:24.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:23 smithi100 bash[31715]: audit 2024-07-28T19:20:22.719224+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:20:24.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:23 smithi138 bash[23952]: audit 2024-07-28T19:20:22.714927+0000 mon.a (mon.0) 305 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-28T19:20:24.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:23 smithi138 bash[23952]: cluster 2024-07-28T19:20:22.714994+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-07-28T19:20:24.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:23 smithi138 bash[23952]: audit 2024-07-28T19:20:22.715237+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:24.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:23 smithi138 bash[23952]: audit 2024-07-28T19:20:22.718575+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:20:24.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:23 smithi138 bash[23952]: audit 2024-07-28T19:20:22.719224+0000 mon.a (mon.0) 308 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:20:25.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[21828]: cluster 2024-07-28T19:20:23.334507+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:25.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[21828]: audit 2024-07-28T19:20:23.727529+0000 mon.a (mon.0) 309 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:20:25.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[21828]: cluster 2024-07-28T19:20:23.727735+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-07-28T19:20:25.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[21828]: audit 2024-07-28T19:20:23.728234+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:25.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[21828]: audit 2024-07-28T19:20:23.735181+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:25.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[31715]: cluster 2024-07-28T19:20:23.334507+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:25.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[31715]: audit 2024-07-28T19:20:23.727529+0000 mon.a (mon.0) 309 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:20:25.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[31715]: cluster 2024-07-28T19:20:23.727735+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-07-28T19:20:25.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[31715]: audit 2024-07-28T19:20:23.728234+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:25.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:24 smithi100 bash[31715]: audit 2024-07-28T19:20:23.735181+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:25.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:24 smithi138 bash[23952]: cluster 2024-07-28T19:20:23.334507+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-07-28T19:20:25.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:24 smithi138 bash[23952]: audit 2024-07-28T19:20:23.727529+0000 mon.a (mon.0) 309 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:20:25.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:24 smithi138 bash[23952]: cluster 2024-07-28T19:20:23.727735+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-07-28T19:20:25.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:24 smithi138 bash[23952]: audit 2024-07-28T19:20:23.728234+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:25.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:24 smithi138 bash[23952]: audit 2024-07-28T19:20:23.735181+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:25.407 INFO:teuthology.orchestra.run.smithi100.stdout:Created osd(s) 1 on host 'smithi100' 2024-07-28T19:20:26.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[21828]: cluster 2024-07-28T19:20:23.352848+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:20:26.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[21828]: cluster 2024-07-28T19:20:23.352935+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:20:26.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[21828]: cluster 2024-07-28T19:20:24.724940+0000 mon.a (mon.0) 313 : cluster [INF] osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596] boot 2024-07-28T19:20:26.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[21828]: cluster 2024-07-28T19:20:24.724992+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-07-28T19:20:26.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[21828]: audit 2024-07-28T19:20:24.725395+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:26.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[21828]: audit 2024-07-28T19:20:25.388346+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:20:26.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[21828]: audit 2024-07-28T19:20:25.395816+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[21828]: audit 2024-07-28T19:20:25.403331+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[31715]: cluster 2024-07-28T19:20:23.352848+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[31715]: cluster 2024-07-28T19:20:23.352935+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[31715]: cluster 2024-07-28T19:20:24.724940+0000 mon.a (mon.0) 313 : cluster [INF] osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596] boot 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[31715]: cluster 2024-07-28T19:20:24.724992+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[31715]: audit 2024-07-28T19:20:24.725395+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[31715]: audit 2024-07-28T19:20:25.388346+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[31715]: audit 2024-07-28T19:20:25.395816+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:26.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:25 smithi100 bash[31715]: audit 2024-07-28T19:20:25.403331+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:26.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:25 smithi138 bash[23952]: cluster 2024-07-28T19:20:23.352848+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:20:26.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:25 smithi138 bash[23952]: cluster 2024-07-28T19:20:23.352935+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:20:26.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:25 smithi138 bash[23952]: cluster 2024-07-28T19:20:24.724940+0000 mon.a (mon.0) 313 : cluster [INF] osd.1 [v2:172.21.15.100:6810/3874116596,v1:172.21.15.100:6811/3874116596] boot 2024-07-28T19:20:26.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:25 smithi138 bash[23952]: cluster 2024-07-28T19:20:24.724992+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-07-28T19:20:26.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:25 smithi138 bash[23952]: audit 2024-07-28T19:20:24.725395+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:20:26.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:25 smithi138 bash[23952]: audit 2024-07-28T19:20:25.388346+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:20:26.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:25 smithi138 bash[23952]: audit 2024-07-28T19:20:25.395816+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:26.060 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:25 smithi138 bash[23952]: audit 2024-07-28T19:20:25.403331+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:26.085 DEBUG:teuthology.orchestra.run.smithi100:osd.1> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.1.service 2024-07-28T19:20:26.087 INFO:tasks.cephadm:Deploying osd.2 on smithi100 with /dev/vg_nvme/lv_2... 2024-07-28T19:20:26.088 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-07-28T19:20:27.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:26 smithi100 bash[21828]: cluster 2024-07-28T19:20:25.335074+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:27.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:26 smithi100 bash[21828]: cluster 2024-07-28T19:20:25.742633+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-07-28T19:20:27.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:26 smithi100 bash[31715]: cluster 2024-07-28T19:20:25.335074+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:27.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:26 smithi100 bash[31715]: cluster 2024-07-28T19:20:25.742633+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-07-28T19:20:27.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:26 smithi138 bash[23952]: cluster 2024-07-28T19:20:25.335074+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:27.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:26 smithi138 bash[23952]: cluster 2024-07-28T19:20:25.742633+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-07-28T19:20:29.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:28 smithi100 bash[21828]: cluster 2024-07-28T19:20:27.335632+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:29.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:28 smithi100 bash[31715]: cluster 2024-07-28T19:20:27.335632+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:29.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:28 smithi138 bash[23952]: cluster 2024-07-28T19:20:27.335632+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:30.166 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:20:30.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:30 smithi100 bash[21828]: cluster 2024-07-28T19:20:29.336202+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:31.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:30 smithi100 bash[31715]: cluster 2024-07-28T19:20:29.336202+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:31.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:30 smithi138 bash[23952]: cluster 2024-07-28T19:20:29.336202+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:32.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:32 smithi100 bash[21828]: cluster 2024-07-28T19:20:31.336774+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:33.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:32 smithi100 bash[31715]: cluster 2024-07-28T19:20:31.336774+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:33.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:32 smithi138 bash[23952]: cluster 2024-07-28T19:20:31.336774+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:34.888 INFO:teuthology.orchestra.run.smithi100.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-07-28T19:20:34.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:34 smithi100 bash[21828]: cluster 2024-07-28T19:20:33.337308+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:35.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:34 smithi100 bash[31715]: cluster 2024-07-28T19:20:33.337308+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:35.047 INFO:teuthology.orchestra.run.smithi100.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-07-28T19:20:35.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:34 smithi138 bash[23952]: cluster 2024-07-28T19:20:33.337308+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:35.062 INFO:teuthology.orchestra.run.smithi100.stderr: stderr: 10+0 records in 2024-07-28T19:20:35.062 INFO:teuthology.orchestra.run.smithi100.stderr:10+0 records out 2024-07-28T19:20:35.062 INFO:teuthology.orchestra.run.smithi100.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.013871 s, 756 MB/s 2024-07-28T19:20:35.063 INFO:teuthology.orchestra.run.smithi100.stderr:--> Zapping successful for: 2024-07-28T19:20:35.983 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch daemon add osd smithi100:vg_nvme/lv_2 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: cluster 2024-07-28T19:20:35.337620+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: audit 2024-07-28T19:20:36.567104+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: audit 2024-07-28T19:20:36.575574+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: audit 2024-07-28T19:20:36.577513+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: audit 2024-07-28T19:20:36.579005+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: audit 2024-07-28T19:20:36.590145+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: audit 2024-07-28T19:20:36.592480+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: audit 2024-07-28T19:20:36.594270+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:20:37.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:36 smithi138 bash[23952]: audit 2024-07-28T19:20:36.606505+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: cluster 2024-07-28T19:20:35.337620+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:37.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: audit 2024-07-28T19:20:36.567104+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: audit 2024-07-28T19:20:36.575574+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: audit 2024-07-28T19:20:36.577513+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:20:37.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: audit 2024-07-28T19:20:36.579005+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:20:37.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: audit 2024-07-28T19:20:36.590145+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: audit 2024-07-28T19:20:36.592480+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: audit 2024-07-28T19:20:36.594270+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[21828]: audit 2024-07-28T19:20:36.606505+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: cluster 2024-07-28T19:20:35.337620+0000 mgr.y (mgr.14150) 145 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: audit 2024-07-28T19:20:36.567104+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: audit 2024-07-28T19:20:36.575574+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: audit 2024-07-28T19:20:36.577513+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: audit 2024-07-28T19:20:36.579005+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: audit 2024-07-28T19:20:36.590145+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: audit 2024-07-28T19:20:36.592480+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: audit 2024-07-28T19:20:36.594270+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:20:37.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:36 smithi100 bash[31715]: audit 2024-07-28T19:20:36.606505+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:20:38.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:37 smithi138 bash[23952]: cephadm 2024-07-28T19:20:36.558299+0000 mgr.y (mgr.14150) 146 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:20:38.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:37 smithi138 bash[23952]: cephadm 2024-07-28T19:20:36.580098+0000 mgr.y (mgr.14150) 147 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 8120M 2024-07-28T19:20:38.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:37 smithi100 bash[21828]: cephadm 2024-07-28T19:20:36.558299+0000 mgr.y (mgr.14150) 146 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:20:38.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:37 smithi100 bash[21828]: cephadm 2024-07-28T19:20:36.580098+0000 mgr.y (mgr.14150) 147 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 8120M 2024-07-28T19:20:38.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:37 smithi100 bash[31715]: cephadm 2024-07-28T19:20:36.558299+0000 mgr.y (mgr.14150) 146 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:20:38.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:37 smithi100 bash[31715]: cephadm 2024-07-28T19:20:36.580098+0000 mgr.y (mgr.14150) 147 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 8120M 2024-07-28T19:20:39.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:38 smithi138 bash[23952]: cluster 2024-07-28T19:20:37.338038+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:39.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:38 smithi100 bash[21828]: cluster 2024-07-28T19:20:37.338038+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:39.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:38 smithi100 bash[31715]: cluster 2024-07-28T19:20:37.338038+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:40.723 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:20:41.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:40 smithi138 bash[23952]: cluster 2024-07-28T19:20:39.338513+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:41.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:40 smithi100 bash[21828]: cluster 2024-07-28T19:20:39.338513+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:41.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:40 smithi100 bash[31715]: cluster 2024-07-28T19:20:39.338513+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:43.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:42 smithi100 bash[21828]: cluster 2024-07-28T19:20:41.338950+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:43.030 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:42 smithi100 bash[31715]: cluster 2024-07-28T19:20:41.338950+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:43.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:42 smithi138 bash[23952]: cluster 2024-07-28T19:20:41.338950+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:44.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:43 smithi138 bash[23952]: audit 2024-07-28T19:20:43.778321+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:20:44.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:43 smithi100 bash[21828]: audit 2024-07-28T19:20:43.778321+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:20:44.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:43 smithi100 bash[31715]: audit 2024-07-28T19:20:43.778321+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:20:45.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:44 smithi138 bash[23952]: cluster 2024-07-28T19:20:43.339357+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:45.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:44 smithi138 bash[23952]: audit 2024-07-28T19:20:43.772247+0000 mgr.y (mgr.14150) 152 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:20:45.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:44 smithi138 bash[23952]: audit 2024-07-28T19:20:43.787914+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:20:45.059 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:44 smithi138 bash[23952]: audit 2024-07-28T19:20:43.789277+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:45.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:44 smithi100 bash[21828]: cluster 2024-07-28T19:20:43.339357+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:45.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:44 smithi100 bash[21828]: audit 2024-07-28T19:20:43.772247+0000 mgr.y (mgr.14150) 152 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:20:45.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:44 smithi100 bash[21828]: audit 2024-07-28T19:20:43.787914+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:20:45.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:44 smithi100 bash[21828]: audit 2024-07-28T19:20:43.789277+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:45.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:44 smithi100 bash[31715]: cluster 2024-07-28T19:20:43.339357+0000 mgr.y (mgr.14150) 151 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:45.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:44 smithi100 bash[31715]: audit 2024-07-28T19:20:43.772247+0000 mgr.y (mgr.14150) 152 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:20:45.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:44 smithi100 bash[31715]: audit 2024-07-28T19:20:43.787914+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:20:45.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:44 smithi100 bash[31715]: audit 2024-07-28T19:20:43.789277+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:20:47.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:46 smithi138 bash[23952]: cluster 2024-07-28T19:20:45.339880+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:47.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:46 smithi100 bash[21828]: cluster 2024-07-28T19:20:45.339880+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:47.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:46 smithi100 bash[31715]: cluster 2024-07-28T19:20:45.339880+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:49.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:48 smithi100 bash[21828]: cluster 2024-07-28T19:20:47.340415+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:49.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:48 smithi100 bash[31715]: cluster 2024-07-28T19:20:47.340415+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:49.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:48 smithi138 bash[23952]: cluster 2024-07-28T19:20:47.340415+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:51.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:50 smithi138 bash[23952]: cluster 2024-07-28T19:20:49.340882+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:51.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:50 smithi100 bash[21828]: cluster 2024-07-28T19:20:49.340882+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:51.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:50 smithi100 bash[31715]: cluster 2024-07-28T19:20:49.340882+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:53.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[21828]: cluster 2024-07-28T19:20:51.341346+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:53.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[21828]: audit 2024-07-28T19:20:52.164384+0000 mon.a (mon.0) 331 : audit [INF] from='client.? 172.21.15.100:0/2322830011' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3416396-83c7-42fb-9153-9cb806281347"}]: dispatch 2024-07-28T19:20:53.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[21828]: audit 2024-07-28T19:20:52.170256+0000 mon.a (mon.0) 332 : audit [INF] from='client.? 172.21.15.100:0/2322830011' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3416396-83c7-42fb-9153-9cb806281347"}]': finished 2024-07-28T19:20:53.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[21828]: cluster 2024-07-28T19:20:52.170317+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-07-28T19:20:53.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[21828]: audit 2024-07-28T19:20:52.170522+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:20:53.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[31715]: cluster 2024-07-28T19:20:51.341346+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:53.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[31715]: audit 2024-07-28T19:20:52.164384+0000 mon.a (mon.0) 331 : audit [INF] from='client.? 172.21.15.100:0/2322830011' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3416396-83c7-42fb-9153-9cb806281347"}]: dispatch 2024-07-28T19:20:53.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[31715]: audit 2024-07-28T19:20:52.170256+0000 mon.a (mon.0) 332 : audit [INF] from='client.? 172.21.15.100:0/2322830011' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3416396-83c7-42fb-9153-9cb806281347"}]': finished 2024-07-28T19:20:53.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[31715]: cluster 2024-07-28T19:20:52.170317+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-07-28T19:20:53.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:52 smithi100 bash[31715]: audit 2024-07-28T19:20:52.170522+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:20:53.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:52 smithi138 bash[23952]: cluster 2024-07-28T19:20:51.341346+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:53.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:52 smithi138 bash[23952]: audit 2024-07-28T19:20:52.164384+0000 mon.a (mon.0) 331 : audit [INF] from='client.? 172.21.15.100:0/2322830011' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3416396-83c7-42fb-9153-9cb806281347"}]: dispatch 2024-07-28T19:20:53.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:52 smithi138 bash[23952]: audit 2024-07-28T19:20:52.170256+0000 mon.a (mon.0) 332 : audit [INF] from='client.? 172.21.15.100:0/2322830011' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3416396-83c7-42fb-9153-9cb806281347"}]': finished 2024-07-28T19:20:53.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:52 smithi138 bash[23952]: cluster 2024-07-28T19:20:52.170317+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-07-28T19:20:53.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:52 smithi138 bash[23952]: audit 2024-07-28T19:20:52.170522+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:20:54.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:53 smithi100 bash[21828]: audit 2024-07-28T19:20:52.814343+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.100:0/2170925447' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:20:54.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:53 smithi100 bash[31715]: audit 2024-07-28T19:20:52.814343+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.100:0/2170925447' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:20:54.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:53 smithi138 bash[23952]: audit 2024-07-28T19:20:52.814343+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.100:0/2170925447' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:20:55.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:54 smithi100 bash[21828]: cluster 2024-07-28T19:20:53.341806+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:55.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:54 smithi100 bash[31715]: cluster 2024-07-28T19:20:53.341806+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:55.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:54 smithi138 bash[23952]: cluster 2024-07-28T19:20:53.341806+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:57.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:56 smithi100 bash[21828]: cluster 2024-07-28T19:20:55.342193+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:57.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:56 smithi100 bash[31715]: cluster 2024-07-28T19:20:55.342193+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:57.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:56 smithi138 bash[23952]: cluster 2024-07-28T19:20:55.342193+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:59.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:20:58 smithi100 bash[21828]: cluster 2024-07-28T19:20:57.342571+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:59.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:20:58 smithi100 bash[31715]: cluster 2024-07-28T19:20:57.342571+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:20:59.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:20:58 smithi138 bash[23952]: cluster 2024-07-28T19:20:57.342571+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:01.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:00 smithi100 bash[21828]: cluster 2024-07-28T19:20:59.342970+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:01.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:00 smithi100 bash[31715]: cluster 2024-07-28T19:20:59.342970+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:01.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:00 smithi138 bash[23952]: cluster 2024-07-28T19:20:59.342970+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:03.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:02 smithi100 bash[21828]: cluster 2024-07-28T19:21:01.343396+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:03.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:02 smithi100 bash[31715]: cluster 2024-07-28T19:21:01.343396+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:03.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:02 smithi138 bash[23952]: cluster 2024-07-28T19:21:01.343396+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:05.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:04 smithi100 bash[21828]: cluster 2024-07-28T19:21:03.343906+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:05.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:04 smithi100 bash[21828]: audit 2024-07-28T19:21:04.555091+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:21:05.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:04 smithi100 bash[21828]: audit 2024-07-28T19:21:04.556812+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:05.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:04 smithi100 bash[31715]: cluster 2024-07-28T19:21:03.343906+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:05.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:04 smithi100 bash[31715]: audit 2024-07-28T19:21:04.555091+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:21:05.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:04 smithi100 bash[31715]: audit 2024-07-28T19:21:04.556812+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:05.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:04 smithi138 bash[23952]: cluster 2024-07-28T19:21:03.343906+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:05.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:04 smithi138 bash[23952]: audit 2024-07-28T19:21:04.555091+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:21:05.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:04 smithi138 bash[23952]: audit 2024-07-28T19:21:04.556812+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:06.158 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:05 smithi100 bash[21828]: cephadm 2024-07-28T19:21:04.558091+0000 mgr.y (mgr.14150) 163 : cephadm [INF] Deploying daemon osd.2 on smithi100 2024-07-28T19:21:06.158 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:05 smithi100 bash[31715]: cephadm 2024-07-28T19:21:04.558091+0000 mgr.y (mgr.14150) 163 : cephadm [INF] Deploying daemon osd.2 on smithi100 2024-07-28T19:21:06.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:05 smithi138 bash[23952]: cephadm 2024-07-28T19:21:04.558091+0000 mgr.y (mgr.14150) 163 : cephadm [INF] Deploying daemon osd.2 on smithi100 2024-07-28T19:21:07.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:06 smithi100 bash[21828]: cluster 2024-07-28T19:21:05.344358+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:07.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:06 smithi100 bash[31715]: cluster 2024-07-28T19:21:05.344358+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:07.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:06 smithi138 bash[23952]: cluster 2024-07-28T19:21:05.344358+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:08.860 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:08.860 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:08.860 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:08 smithi100 bash[31715]: cluster 2024-07-28T19:21:07.344811+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:08.861 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:08.861 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:08.861 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:08.861 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:08 smithi100 bash[21828]: cluster 2024-07-28T19:21:07.344811+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:09.200 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:09.200 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:09.200 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:09.201 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:09.201 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:21:08 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:21:09.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:08 smithi138 bash[23952]: cluster 2024-07-28T19:21:07.344811+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:10.117 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:09 smithi100 bash[31715]: audit 2024-07-28T19:21:09.123899+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:21:10.117 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:09 smithi100 bash[31715]: audit 2024-07-28T19:21:09.127841+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:10.118 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:09 smithi100 bash[31715]: audit 2024-07-28T19:21:09.133808+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:10.118 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:09 smithi100 bash[21828]: audit 2024-07-28T19:21:09.123899+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:21:10.118 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:09 smithi100 bash[21828]: audit 2024-07-28T19:21:09.127841+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:10.118 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:09 smithi100 bash[21828]: audit 2024-07-28T19:21:09.133808+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:10.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:09 smithi138 bash[23952]: audit 2024-07-28T19:21:09.123899+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:21:10.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:09 smithi138 bash[23952]: audit 2024-07-28T19:21:09.127841+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:10.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:09 smithi138 bash[23952]: audit 2024-07-28T19:21:09.133808+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:11.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:11 smithi100 bash[21828]: cluster 2024-07-28T19:21:09.345156+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:11.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:11 smithi100 bash[31715]: cluster 2024-07-28T19:21:09.345156+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:11.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:11 smithi138 bash[23952]: cluster 2024-07-28T19:21:09.345156+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:13.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:13 smithi100 bash[21828]: cluster 2024-07-28T19:21:11.345454+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:13.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:13 smithi100 bash[31715]: cluster 2024-07-28T19:21:11.345454+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:13.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:13 smithi138 bash[23952]: cluster 2024-07-28T19:21:11.345454+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:14.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:14 smithi100 bash[21828]: cluster 2024-07-28T19:21:13.345882+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:14.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:14 smithi100 bash[31715]: cluster 2024-07-28T19:21:13.345882+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:14.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:14 smithi138 bash[23952]: cluster 2024-07-28T19:21:13.345882+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:15.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[21828]: audit 2024-07-28T19:21:14.172712+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:15.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[21828]: audit 2024-07-28T19:21:14.176791+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:15.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[21828]: audit 2024-07-28T19:21:14.559004+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:15.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[21828]: audit 2024-07-28T19:21:14.560596+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:21:15.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[21828]: audit 2024-07-28T19:21:14.569773+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:15.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[31715]: audit 2024-07-28T19:21:14.172712+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:15.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[31715]: audit 2024-07-28T19:21:14.176791+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:15.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[31715]: audit 2024-07-28T19:21:14.559004+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:15.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[31715]: audit 2024-07-28T19:21:14.560596+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:21:15.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:15 smithi100 bash[31715]: audit 2024-07-28T19:21:14.569773+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:15.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:15 smithi138 bash[23952]: audit 2024-07-28T19:21:14.172712+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:15.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:15 smithi138 bash[23952]: audit 2024-07-28T19:21:14.176791+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:15.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:15 smithi138 bash[23952]: audit 2024-07-28T19:21:14.559004+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:15.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:15 smithi138 bash[23952]: audit 2024-07-28T19:21:14.560596+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:21:15.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:15 smithi138 bash[23952]: audit 2024-07-28T19:21:14.569773+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:16.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:16 smithi100 bash[21828]: cluster 2024-07-28T19:21:15.346285+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:16.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:16 smithi100 bash[21828]: audit 2024-07-28T19:21:15.833365+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:21:16.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:16 smithi100 bash[21828]: audit 2024-07-28T19:21:15.833693+0000 mon.a (mon.0) 345 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:21:16.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:16 smithi100 bash[31715]: cluster 2024-07-28T19:21:15.346285+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:16.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:16 smithi100 bash[31715]: audit 2024-07-28T19:21:15.833365+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:21:16.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:16 smithi100 bash[31715]: audit 2024-07-28T19:21:15.833693+0000 mon.a (mon.0) 345 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:21:16.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:16 smithi138 bash[23952]: cluster 2024-07-28T19:21:15.346285+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:16.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:16 smithi138 bash[23952]: audit 2024-07-28T19:21:15.833365+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:21:16.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:16 smithi138 bash[23952]: audit 2024-07-28T19:21:15.833693+0000 mon.a (mon.0) 345 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:21:17.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[21828]: audit 2024-07-28T19:21:16.187987+0000 mon.a (mon.0) 346 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-28T19:21:17.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[21828]: cluster 2024-07-28T19:21:16.188172+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-07-28T19:21:17.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[21828]: audit 2024-07-28T19:21:16.188467+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:17.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[21828]: audit 2024-07-28T19:21:16.189630+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:21:17.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[21828]: audit 2024-07-28T19:21:16.190251+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:21:17.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[31715]: audit 2024-07-28T19:21:16.187987+0000 mon.a (mon.0) 346 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-28T19:21:17.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[31715]: cluster 2024-07-28T19:21:16.188172+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-07-28T19:21:17.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[31715]: audit 2024-07-28T19:21:16.188467+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:17.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[31715]: audit 2024-07-28T19:21:16.189630+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:21:17.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:17 smithi100 bash[31715]: audit 2024-07-28T19:21:16.190251+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:21:17.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:17 smithi138 bash[23952]: audit 2024-07-28T19:21:16.187987+0000 mon.a (mon.0) 346 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-28T19:21:17.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:17 smithi138 bash[23952]: cluster 2024-07-28T19:21:16.188172+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-07-28T19:21:17.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:17 smithi138 bash[23952]: audit 2024-07-28T19:21:16.188467+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:17.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:17 smithi138 bash[23952]: audit 2024-07-28T19:21:16.189630+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:21:17.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:17 smithi138 bash[23952]: audit 2024-07-28T19:21:16.190251+0000 mon.a (mon.0) 349 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:21:18.025 INFO:teuthology.orchestra.run.smithi100.stdout:Created osd(s) 2 on host 'smithi100' 2024-07-28T19:21:18.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[21828]: audit 2024-07-28T19:21:17.188791+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[21828]: cluster 2024-07-28T19:21:17.188908+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[21828]: audit 2024-07-28T19:21:17.189615+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[21828]: audit 2024-07-28T19:21:17.198250+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[21828]: cluster 2024-07-28T19:21:17.346619+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v136: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[21828]: audit 2024-07-28T19:21:18.003348+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[21828]: audit 2024-07-28T19:21:18.010683+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[21828]: audit 2024-07-28T19:21:18.019321+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[31715]: audit 2024-07-28T19:21:17.188791+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:21:18.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[31715]: cluster 2024-07-28T19:21:17.188908+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-07-28T19:21:18.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[31715]: audit 2024-07-28T19:21:17.189615+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:18.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[31715]: audit 2024-07-28T19:21:17.198250+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:18.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[31715]: cluster 2024-07-28T19:21:17.346619+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v136: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:18.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[31715]: audit 2024-07-28T19:21:18.003348+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:21:18.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[31715]: audit 2024-07-28T19:21:18.010683+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:18.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:18 smithi100 bash[31715]: audit 2024-07-28T19:21:18.019321+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:18.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:18 smithi138 bash[23952]: audit 2024-07-28T19:21:17.188791+0000 mon.a (mon.0) 350 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:21:18.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:18 smithi138 bash[23952]: cluster 2024-07-28T19:21:17.188908+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-07-28T19:21:18.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:18 smithi138 bash[23952]: audit 2024-07-28T19:21:17.189615+0000 mon.a (mon.0) 352 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:18.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:18 smithi138 bash[23952]: audit 2024-07-28T19:21:17.198250+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:18.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:18 smithi138 bash[23952]: cluster 2024-07-28T19:21:17.346619+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v136: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-07-28T19:21:18.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:18 smithi138 bash[23952]: audit 2024-07-28T19:21:18.003348+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:21:18.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:18 smithi138 bash[23952]: audit 2024-07-28T19:21:18.010683+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:18.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:18 smithi138 bash[23952]: audit 2024-07-28T19:21:18.019321+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:18.734 DEBUG:teuthology.orchestra.run.smithi100:osd.2> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.2.service 2024-07-28T19:21:18.737 INFO:tasks.cephadm:Deploying osd.3 on smithi100 with /dev/vg_nvme/lv_1... 2024-07-28T19:21:18.737 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-07-28T19:21:19.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[21828]: cluster 2024-07-28T19:21:16.853523+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[21828]: cluster 2024-07-28T19:21:16.853637+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[21828]: audit 2024-07-28T19:21:18.197214+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[21828]: cluster 2024-07-28T19:21:18.210141+0000 mon.a (mon.0) 358 : cluster [INF] osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255] boot 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[21828]: cluster 2024-07-28T19:21:18.210196+0000 mon.a (mon.0) 359 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[21828]: audit 2024-07-28T19:21:18.210778+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[31715]: cluster 2024-07-28T19:21:16.853523+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[31715]: cluster 2024-07-28T19:21:16.853637+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[31715]: audit 2024-07-28T19:21:18.197214+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:19.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[31715]: cluster 2024-07-28T19:21:18.210141+0000 mon.a (mon.0) 358 : cluster [INF] osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255] boot 2024-07-28T19:21:19.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[31715]: cluster 2024-07-28T19:21:18.210196+0000 mon.a (mon.0) 359 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-07-28T19:21:19.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:19 smithi100 bash[31715]: audit 2024-07-28T19:21:18.210778+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:19.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:19 smithi138 bash[23952]: cluster 2024-07-28T19:21:16.853523+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:21:19.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:19 smithi138 bash[23952]: cluster 2024-07-28T19:21:16.853637+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:21:19.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:19 smithi138 bash[23952]: audit 2024-07-28T19:21:18.197214+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:19.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:19 smithi138 bash[23952]: cluster 2024-07-28T19:21:18.210141+0000 mon.a (mon.0) 358 : cluster [INF] osd.2 [v2:172.21.15.100:6818/2139838255,v1:172.21.15.100:6819/2139838255] boot 2024-07-28T19:21:19.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:19 smithi138 bash[23952]: cluster 2024-07-28T19:21:18.210196+0000 mon.a (mon.0) 359 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-07-28T19:21:19.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:19 smithi138 bash[23952]: audit 2024-07-28T19:21:18.210778+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:21:20.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:20 smithi100 bash[21828]: cluster 2024-07-28T19:21:19.210521+0000 mon.a (mon.0) 361 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-07-28T19:21:20.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:20 smithi100 bash[21828]: cluster 2024-07-28T19:21:19.347045+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v139: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:20.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:20 smithi100 bash[21828]: audit 2024-07-28T19:21:19.449409+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-07-28T19:21:20.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:20 smithi100 bash[31715]: cluster 2024-07-28T19:21:19.210521+0000 mon.a (mon.0) 361 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-07-28T19:21:20.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:20 smithi100 bash[31715]: cluster 2024-07-28T19:21:19.347045+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v139: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:20.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:20 smithi100 bash[31715]: audit 2024-07-28T19:21:19.449409+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-07-28T19:21:20.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:20 smithi138 bash[23952]: cluster 2024-07-28T19:21:19.210521+0000 mon.a (mon.0) 361 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-07-28T19:21:20.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:20 smithi138 bash[23952]: cluster 2024-07-28T19:21:19.347045+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v139: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:20.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:20 smithi138 bash[23952]: audit 2024-07-28T19:21:19.449409+0000 mon.a (mon.0) 362 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-07-28T19:21:21.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:21 smithi100 bash[21828]: audit 2024-07-28T19:21:20.234252+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-07-28T19:21:21.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:21 smithi100 bash[21828]: cluster 2024-07-28T19:21:20.234431+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-07-28T19:21:21.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:21 smithi100 bash[21828]: audit 2024-07-28T19:21:20.236719+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:21:21.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:21 smithi100 bash[31715]: audit 2024-07-28T19:21:20.234252+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-07-28T19:21:21.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:21 smithi100 bash[31715]: cluster 2024-07-28T19:21:20.234431+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-07-28T19:21:21.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:21 smithi100 bash[31715]: audit 2024-07-28T19:21:20.236719+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:21:21.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:21 smithi138 bash[23952]: audit 2024-07-28T19:21:20.234252+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-07-28T19:21:21.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:21 smithi138 bash[23952]: cluster 2024-07-28T19:21:20.234431+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-07-28T19:21:21.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:21 smithi138 bash[23952]: audit 2024-07-28T19:21:20.236719+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:21:22.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:22 smithi100 bash[21828]: audit 2024-07-28T19:21:21.232167+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-07-28T19:21:22.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:22 smithi100 bash[21828]: cluster 2024-07-28T19:21:21.232303+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-07-28T19:21:22.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:22 smithi100 bash[21828]: cluster 2024-07-28T19:21:21.347559+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v142: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:22.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:22 smithi100 bash[31715]: audit 2024-07-28T19:21:21.232167+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-07-28T19:21:22.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:22 smithi100 bash[31715]: cluster 2024-07-28T19:21:21.232303+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-07-28T19:21:22.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:22 smithi100 bash[31715]: cluster 2024-07-28T19:21:21.347559+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v142: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:22.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:22 smithi138 bash[23952]: audit 2024-07-28T19:21:21.232167+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-07-28T19:21:22.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:22 smithi138 bash[23952]: cluster 2024-07-28T19:21:21.232303+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-07-28T19:21:22.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:22 smithi138 bash[23952]: cluster 2024-07-28T19:21:21.347559+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v142: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:22.822 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:21:23.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:23 smithi138 bash[23952]: cluster 2024-07-28T19:21:22.252737+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-07-28T19:21:23.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:23 smithi138 bash[23952]: audit 2024-07-28T19:21:22.723280+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:23.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:23 smithi100 bash[31715]: cluster 2024-07-28T19:21:22.252737+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-07-28T19:21:23.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:23 smithi100 bash[31715]: audit 2024-07-28T19:21:22.723280+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:23.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:23 smithi100 bash[21828]: cluster 2024-07-28T19:21:22.252737+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2024-07-28T19:21:23.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:23 smithi100 bash[21828]: audit 2024-07-28T19:21:22.723280+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:24.554 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:24 smithi100 bash[21828]: cluster 2024-07-28T19:21:23.347988+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v144: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:24.554 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:24 smithi100 bash[31715]: cluster 2024-07-28T19:21:23.347988+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v144: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:24.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:24 smithi138 bash[23952]: cluster 2024-07-28T19:21:23.347988+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v144: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:25.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:25 smithi100 bash[31715]: cluster 2024-07-28T19:21:24.265342+0000 mon.a (mon.0) 370 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-07-28T19:21:25.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:25 smithi100 bash[31715]: audit 2024-07-28T19:21:24.478019+0000 mon.a (mon.0) 371 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:25.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:25 smithi100 bash[21828]: cluster 2024-07-28T19:21:24.265342+0000 mon.a (mon.0) 370 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-07-28T19:21:25.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:25 smithi100 bash[21828]: audit 2024-07-28T19:21:24.478019+0000 mon.a (mon.0) 371 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:25.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:25 smithi138 bash[23952]: cluster 2024-07-28T19:21:24.265342+0000 mon.a (mon.0) 370 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-07-28T19:21:25.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:25 smithi138 bash[23952]: audit 2024-07-28T19:21:24.478019+0000 mon.a (mon.0) 371 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:25.334686+0000 mon.a (mon.0) 372 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:25.335406+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:25.335580+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:25.335779+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:25.343424+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:25.343698+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:25.343814+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:25.343990+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: cluster 2024-07-28T19:21:25.348329+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:26.245382+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:26.282126+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:27.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:26.285151+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:26.285528+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[21828]: audit 2024-07-28T19:21:26.285924+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:25.334686+0000 mon.a (mon.0) 372 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:25.335406+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:25.335580+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:25.335779+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:25.343424+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:25.343698+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:25.343814+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:25.343990+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: cluster 2024-07-28T19:21:25.348329+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:27.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:26.245382+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:27.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:26.282126+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:27.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:26.285151+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:26.285528+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:27 smithi100 bash[31715]: audit 2024-07-28T19:21:26.285924+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:25.334686+0000 mon.a (mon.0) 372 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:25.335406+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:25.335580+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:25.335779+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:25.343424+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:25.343698+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:25.343814+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:25.343990+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: cluster 2024-07-28T19:21:25.348329+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:26.245382+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:26.282126+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:26.285151+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:26.285528+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:21:27.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:27 smithi138 bash[23952]: audit 2024-07-28T19:21:26.285924+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:21:27.895 INFO:teuthology.orchestra.run.smithi100.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-07-28T19:21:28.062 INFO:teuthology.orchestra.run.smithi100.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-07-28T19:21:28.078 INFO:teuthology.orchestra.run.smithi100.stderr: stderr: 10+0 records in 2024-07-28T19:21:28.078 INFO:teuthology.orchestra.run.smithi100.stderr:10+0 records out 2024-07-28T19:21:28.078 INFO:teuthology.orchestra.run.smithi100.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.014197 s, 739 MB/s 2024-07-28T19:21:28.078 INFO:teuthology.orchestra.run.smithi100.stderr:--> Zapping successful for: 2024-07-28T19:21:28.368 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:28 smithi100 bash[21828]: audit 2024-07-28T19:21:26.284685+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:28.368 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:28 smithi100 bash[21828]: audit 2024-07-28T19:21:27.194559+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:28.368 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:28 smithi100 bash[21828]: cluster 2024-07-28T19:21:27.348771+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:28.369 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:28 smithi100 bash[31715]: audit 2024-07-28T19:21:26.284685+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:28.369 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:28 smithi100 bash[31715]: audit 2024-07-28T19:21:27.194559+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:28.369 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:28 smithi100 bash[31715]: cluster 2024-07-28T19:21:27.348771+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:28.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:28 smithi138 bash[23952]: audit 2024-07-28T19:21:26.284685+0000 mon.c (mon.2) 11 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-07-28T19:21:28.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:28 smithi138 bash[23952]: audit 2024-07-28T19:21:27.194559+0000 mon.c (mon.2) 12 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-07-28T19:21:28.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:28 smithi138 bash[23952]: cluster 2024-07-28T19:21:27.348771+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:28.862 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch daemon add osd smithi100:vg_nvme/lv_1 2024-07-28T19:21:30.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: cephadm 2024-07-28T19:21:29.235483+0000 mgr.y (mgr.14150) 176 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.242766+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.252925+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.254507+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.255732+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.256879+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: cephadm 2024-07-28T19:21:29.257795+0000 mgr.y (mgr.14150) 177 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 5413M 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.268296+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.270724+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.272336+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: audit 2024-07-28T19:21:29.279951+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[21828]: cluster 2024-07-28T19:21:29.349329+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:30.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: cephadm 2024-07-28T19:21:29.235483+0000 mgr.y (mgr.14150) 176 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.242766+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.252925+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.254507+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.255732+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.256879+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: cephadm 2024-07-28T19:21:29.257795+0000 mgr.y (mgr.14150) 177 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 5413M 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.268296+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.270724+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.272336+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: audit 2024-07-28T19:21:29.279951+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:30 smithi100 bash[31715]: cluster 2024-07-28T19:21:29.349329+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:30.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: cephadm 2024-07-28T19:21:29.235483+0000 mgr.y (mgr.14150) 176 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:21:30.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.242766+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.252925+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.254507+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.255732+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.256879+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:21:30.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: cephadm 2024-07-28T19:21:29.257795+0000 mgr.y (mgr.14150) 177 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 5413M 2024-07-28T19:21:30.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.268296+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.270724+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:30.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.272336+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:21:30.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: audit 2024-07-28T19:21:29.279951+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:21:30.559 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:30 smithi138 bash[23952]: cluster 2024-07-28T19:21:29.349329+0000 mgr.y (mgr.14150) 178 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:32.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:32 smithi100 bash[21828]: cluster 2024-07-28T19:21:31.349866+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-07-28T19:21:32.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:32 smithi100 bash[31715]: cluster 2024-07-28T19:21:31.349866+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-07-28T19:21:32.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:32 smithi138 bash[23952]: cluster 2024-07-28T19:21:31.349866+0000 mgr.y (mgr.14150) 179 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-07-28T19:21:33.610 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:21:34.695 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:34 smithi100 bash[21828]: cluster 2024-07-28T19:21:33.350408+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-07-28T19:21:34.695 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:34 smithi100 bash[31715]: cluster 2024-07-28T19:21:33.350408+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-07-28T19:21:34.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:34 smithi138 bash[23952]: cluster 2024-07-28T19:21:33.350408+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-07-28T19:21:36.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:36 smithi100 bash[21828]: cluster 2024-07-28T19:21:35.351054+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-07-28T19:21:36.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:36 smithi100 bash[31715]: cluster 2024-07-28T19:21:35.351054+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-07-28T19:21:36.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:36 smithi138 bash[23952]: cluster 2024-07-28T19:21:35.351054+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-07-28T19:21:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:37 smithi100 bash[21828]: audit 2024-07-28T19:21:36.758107+0000 mgr.y (mgr.14150) 182 : audit [DBG] from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:21:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:37 smithi100 bash[21828]: audit 2024-07-28T19:21:36.762347+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:21:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:37 smithi100 bash[21828]: audit 2024-07-28T19:21:36.767324+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:21:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:37 smithi100 bash[21828]: audit 2024-07-28T19:21:36.768452+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:37.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:37 smithi100 bash[31715]: audit 2024-07-28T19:21:36.758107+0000 mgr.y (mgr.14150) 182 : audit [DBG] from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:21:37.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:37 smithi100 bash[31715]: audit 2024-07-28T19:21:36.762347+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:21:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:37 smithi100 bash[31715]: audit 2024-07-28T19:21:36.767324+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:21:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:37 smithi100 bash[31715]: audit 2024-07-28T19:21:36.768452+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:37 smithi138 bash[23952]: audit 2024-07-28T19:21:36.758107+0000 mgr.y (mgr.14150) 182 : audit [DBG] from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:21:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:37 smithi138 bash[23952]: audit 2024-07-28T19:21:36.762347+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:21:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:37 smithi138 bash[23952]: audit 2024-07-28T19:21:36.767324+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:21:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:37 smithi138 bash[23952]: audit 2024-07-28T19:21:36.768452+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:38.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:38 smithi100 bash[21828]: cluster 2024-07-28T19:21:37.351596+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:38.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:38 smithi100 bash[31715]: cluster 2024-07-28T19:21:37.351596+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:38 smithi138 bash[23952]: cluster 2024-07-28T19:21:37.351596+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:40.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:40 smithi100 bash[21828]: cluster 2024-07-28T19:21:39.352117+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:40.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:40 smithi100 bash[31715]: cluster 2024-07-28T19:21:39.352117+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:40.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:40 smithi138 bash[23952]: cluster 2024-07-28T19:21:39.352117+0000 mgr.y (mgr.14150) 184 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:42.707 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:42 smithi100 bash[21828]: cluster 2024-07-28T19:21:41.352651+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:42.708 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:42 smithi100 bash[31715]: cluster 2024-07-28T19:21:41.352651+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:42.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:42 smithi138 bash[23952]: cluster 2024-07-28T19:21:41.352651+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-07-28T19:21:44.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:44 smithi100 bash[31715]: cluster 2024-07-28T19:21:43.353150+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:44.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:44 smithi100 bash[21828]: cluster 2024-07-28T19:21:43.353150+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:44.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:44 smithi138 bash[23952]: cluster 2024-07-28T19:21:43.353150+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:45.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[21828]: audit 2024-07-28T19:21:45.049083+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.100:0/2860927890' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]: dispatch 2024-07-28T19:21:45.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[21828]: audit 2024-07-28T19:21:45.049601+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]: dispatch 2024-07-28T19:21:45.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[21828]: audit 2024-07-28T19:21:45.059745+0000 mon.a (mon.0) 396 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]': finished 2024-07-28T19:21:45.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[21828]: cluster 2024-07-28T19:21:45.059852+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-07-28T19:21:45.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[21828]: audit 2024-07-28T19:21:45.060056+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:21:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[31715]: audit 2024-07-28T19:21:45.049083+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.100:0/2860927890' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]: dispatch 2024-07-28T19:21:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[31715]: audit 2024-07-28T19:21:45.049601+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]: dispatch 2024-07-28T19:21:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[31715]: audit 2024-07-28T19:21:45.059745+0000 mon.a (mon.0) 396 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]': finished 2024-07-28T19:21:45.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[31715]: cluster 2024-07-28T19:21:45.059852+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-07-28T19:21:45.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:45 smithi100 bash[31715]: audit 2024-07-28T19:21:45.060056+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:21:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:45 smithi138 bash[23952]: audit 2024-07-28T19:21:45.049083+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.100:0/2860927890' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]: dispatch 2024-07-28T19:21:45.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:45 smithi138 bash[23952]: audit 2024-07-28T19:21:45.049601+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]: dispatch 2024-07-28T19:21:45.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:45 smithi138 bash[23952]: audit 2024-07-28T19:21:45.059745+0000 mon.a (mon.0) 396 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0363a205-8262-42a4-aaaf-e52558fec7f3"}]': finished 2024-07-28T19:21:45.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:45 smithi138 bash[23952]: cluster 2024-07-28T19:21:45.059852+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-07-28T19:21:45.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:45 smithi138 bash[23952]: audit 2024-07-28T19:21:45.060056+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:21:46.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:46 smithi100 bash[21828]: cluster 2024-07-28T19:21:45.353745+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:46.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:46 smithi100 bash[21828]: audit 2024-07-28T19:21:45.698301+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.100:0/1943772780' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:21:46.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:46 smithi100 bash[31715]: cluster 2024-07-28T19:21:45.353745+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:46.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:46 smithi100 bash[31715]: audit 2024-07-28T19:21:45.698301+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.100:0/1943772780' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:21:46.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:46 smithi138 bash[23952]: cluster 2024-07-28T19:21:45.353745+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:46.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:46 smithi138 bash[23952]: audit 2024-07-28T19:21:45.698301+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.100:0/1943772780' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:21:48.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:48 smithi100 bash[21828]: cluster 2024-07-28T19:21:47.354303+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:48.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:48 smithi100 bash[31715]: cluster 2024-07-28T19:21:47.354303+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:48.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:48 smithi138 bash[23952]: cluster 2024-07-28T19:21:47.354303+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:50 smithi100 bash[21828]: cluster 2024-07-28T19:21:49.354828+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:50.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:50 smithi100 bash[31715]: cluster 2024-07-28T19:21:49.354828+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:50 smithi138 bash[23952]: cluster 2024-07-28T19:21:49.354828+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:52.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:52 smithi100 bash[21828]: cluster 2024-07-28T19:21:51.355388+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:52.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:52 smithi100 bash[31715]: cluster 2024-07-28T19:21:51.355388+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:52.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:52 smithi138 bash[23952]: cluster 2024-07-28T19:21:51.355388+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:54.709 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:54 smithi100 bash[21828]: cluster 2024-07-28T19:21:53.355924+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:54.709 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:54 smithi100 bash[31715]: cluster 2024-07-28T19:21:53.355924+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:54.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:54 smithi138 bash[23952]: cluster 2024-07-28T19:21:53.355924+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:56.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:56 smithi100 bash[21828]: cluster 2024-07-28T19:21:55.356458+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:56.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:56 smithi100 bash[31715]: cluster 2024-07-28T19:21:55.356458+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:56.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:56 smithi138 bash[23952]: cluster 2024-07-28T19:21:55.356458+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:57.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:57 smithi100 bash[31715]: audit 2024-07-28T19:21:57.446487+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:21:57.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:57 smithi100 bash[31715]: audit 2024-07-28T19:21:57.448203+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:57.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:57 smithi100 bash[21828]: audit 2024-07-28T19:21:57.446487+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:21:57.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:57 smithi100 bash[21828]: audit 2024-07-28T19:21:57.448203+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:57.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:57 smithi138 bash[23952]: audit 2024-07-28T19:21:57.446487+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:21:57.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:57 smithi138 bash[23952]: audit 2024-07-28T19:21:57.448203+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:21:58.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:58 smithi100 bash[21828]: cluster 2024-07-28T19:21:57.356969+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:58.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:21:58 smithi100 bash[21828]: cephadm 2024-07-28T19:21:57.449542+0000 mgr.y (mgr.14150) 194 : cephadm [INF] Deploying daemon osd.3 on smithi100 2024-07-28T19:21:58.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:58 smithi100 bash[31715]: cluster 2024-07-28T19:21:57.356969+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:58.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:21:58 smithi100 bash[31715]: cephadm 2024-07-28T19:21:57.449542+0000 mgr.y (mgr.14150) 194 : cephadm [INF] Deploying daemon osd.3 on smithi100 2024-07-28T19:21:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:58 smithi138 bash[23952]: cluster 2024-07-28T19:21:57.356969+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:21:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:21:58 smithi138 bash[23952]: cephadm 2024-07-28T19:21:57.449542+0000 mgr.y (mgr.14150) 194 : cephadm [INF] Deploying daemon osd.3 on smithi100 2024-07-28T19:22:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:00 smithi100 bash[21828]: cluster 2024-07-28T19:21:59.357470+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:00 smithi100 bash[31715]: cluster 2024-07-28T19:21:59.357470+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:00 smithi138 bash[23952]: cluster 2024-07-28T19:21:59.357470+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:01.585 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.586 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.586 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.586 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.587 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.587 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.914 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.915 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.915 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.915 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.916 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:01.916 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:22:01 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:02.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:02 smithi100 bash[21828]: cluster 2024-07-28T19:22:01.358031+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:02.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:02 smithi100 bash[21828]: audit 2024-07-28T19:22:01.833112+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:02.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:02 smithi100 bash[21828]: audit 2024-07-28T19:22:01.837927+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:02.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:02 smithi100 bash[21828]: audit 2024-07-28T19:22:01.843362+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:02 smithi100 bash[31715]: cluster 2024-07-28T19:22:01.358031+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:02 smithi100 bash[31715]: audit 2024-07-28T19:22:01.833112+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:02 smithi100 bash[31715]: audit 2024-07-28T19:22:01.837927+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:02 smithi100 bash[31715]: audit 2024-07-28T19:22:01.843362+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:02.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:02 smithi138 bash[23952]: cluster 2024-07-28T19:22:01.358031+0000 mgr.y (mgr.14150) 196 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:02.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:02 smithi138 bash[23952]: audit 2024-07-28T19:22:01.833112+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:02.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:02 smithi138 bash[23952]: audit 2024-07-28T19:22:01.837927+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:02.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:02 smithi138 bash[23952]: audit 2024-07-28T19:22:01.843362+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:04.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:04 smithi100 bash[21828]: cluster 2024-07-28T19:22:03.358450+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:04.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:04 smithi100 bash[31715]: cluster 2024-07-28T19:22:03.358450+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:04.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:04 smithi138 bash[23952]: cluster 2024-07-28T19:22:03.358450+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:06.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:06 smithi100 bash[21828]: cluster 2024-07-28T19:22:05.358873+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:06.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:06 smithi100 bash[31715]: cluster 2024-07-28T19:22:05.358873+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:06.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:06 smithi138 bash[23952]: cluster 2024-07-28T19:22:05.358873+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:08.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[31715]: audit 2024-07-28T19:22:06.929168+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:08.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[31715]: audit 2024-07-28T19:22:06.933347+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:08.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[31715]: audit 2024-07-28T19:22:06.934103+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:08.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[31715]: audit 2024-07-28T19:22:06.934799+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:08.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[31715]: audit 2024-07-28T19:22:06.939207+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:08.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[21828]: audit 2024-07-28T19:22:06.929168+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:08.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[21828]: audit 2024-07-28T19:22:06.933347+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:08.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[21828]: audit 2024-07-28T19:22:06.934103+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:08.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[21828]: audit 2024-07-28T19:22:06.934799+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:08.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:07 smithi100 bash[21828]: audit 2024-07-28T19:22:06.939207+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:08.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:07 smithi138 bash[23952]: audit 2024-07-28T19:22:06.929168+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:08.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:07 smithi138 bash[23952]: audit 2024-07-28T19:22:06.933347+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:08.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:07 smithi138 bash[23952]: audit 2024-07-28T19:22:06.934103+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:08.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:07 smithi138 bash[23952]: audit 2024-07-28T19:22:06.934799+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:08.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:07 smithi138 bash[23952]: audit 2024-07-28T19:22:06.939207+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:09.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:08 smithi100 bash[21828]: cluster 2024-07-28T19:22:07.359433+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:09.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:08 smithi100 bash[21828]: audit 2024-07-28T19:22:08.559276+0000 mon.c (mon.2) 15 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:22:09.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:08 smithi100 bash[21828]: audit 2024-07-28T19:22:08.559651+0000 mon.a (mon.0) 409 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:22:09.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:08 smithi100 bash[31715]: cluster 2024-07-28T19:22:07.359433+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:09.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:08 smithi100 bash[31715]: audit 2024-07-28T19:22:08.559276+0000 mon.c (mon.2) 15 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:22:09.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:08 smithi100 bash[31715]: audit 2024-07-28T19:22:08.559651+0000 mon.a (mon.0) 409 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:22:09.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:08 smithi138 bash[23952]: cluster 2024-07-28T19:22:07.359433+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:09.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:08 smithi138 bash[23952]: audit 2024-07-28T19:22:08.559276+0000 mon.c (mon.2) 15 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:22:09.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:08 smithi138 bash[23952]: audit 2024-07-28T19:22:08.559651+0000 mon.a (mon.0) 409 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:22:09.976 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[31715]: audit 2024-07-28T19:22:08.946781+0000 mon.a (mon.0) 410 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-28T19:22:09.976 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[31715]: cluster 2024-07-28T19:22:08.946898+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-07-28T19:22:09.977 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[31715]: audit 2024-07-28T19:22:08.947532+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:09.977 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[31715]: audit 2024-07-28T19:22:08.948812+0000 mon.c (mon.2) 16 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:22:09.977 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[31715]: audit 2024-07-28T19:22:08.956095+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:22:09.977 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[21828]: audit 2024-07-28T19:22:08.946781+0000 mon.a (mon.0) 410 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-28T19:22:09.977 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[21828]: cluster 2024-07-28T19:22:08.946898+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-07-28T19:22:09.977 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[21828]: audit 2024-07-28T19:22:08.947532+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:09.977 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[21828]: audit 2024-07-28T19:22:08.948812+0000 mon.c (mon.2) 16 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:22:09.977 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:09 smithi100 bash[21828]: audit 2024-07-28T19:22:08.956095+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:22:10.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:09 smithi138 bash[23952]: audit 2024-07-28T19:22:08.946781+0000 mon.a (mon.0) 410 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-28T19:22:10.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:09 smithi138 bash[23952]: cluster 2024-07-28T19:22:08.946898+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-07-28T19:22:10.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:09 smithi138 bash[23952]: audit 2024-07-28T19:22:08.947532+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:10.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:09 smithi138 bash[23952]: audit 2024-07-28T19:22:08.948812+0000 mon.c (mon.2) 16 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:22:10.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:09 smithi138 bash[23952]: audit 2024-07-28T19:22:08.956095+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:22:10.348 INFO:teuthology.orchestra.run.smithi100.stdout:Created osd(s) 3 on host 'smithi100' 2024-07-28T19:22:10.940 DEBUG:teuthology.orchestra.run.smithi100:osd.3> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.3.service 2024-07-28T19:22:10.942 INFO:tasks.cephadm:Deploying osd.4 on smithi138 with /dev/vg_nvme/lv_4... 2024-07-28T19:22:10.942 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-07-28T19:22:11.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[21828]: cluster 2024-07-28T19:22:09.359830+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[21828]: audit 2024-07-28T19:22:09.965324+0000 mon.a (mon.0) 414 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[21828]: cluster 2024-07-28T19:22:09.965458+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[21828]: audit 2024-07-28T19:22:09.965896+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[21828]: audit 2024-07-28T19:22:09.972923+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[21828]: audit 2024-07-28T19:22:10.327774+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[21828]: audit 2024-07-28T19:22:10.335430+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[21828]: audit 2024-07-28T19:22:10.345075+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[31715]: cluster 2024-07-28T19:22:09.359830+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:11.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[31715]: audit 2024-07-28T19:22:09.965324+0000 mon.a (mon.0) 414 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:22:11.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[31715]: cluster 2024-07-28T19:22:09.965458+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-07-28T19:22:11.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[31715]: audit 2024-07-28T19:22:09.965896+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:11.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[31715]: audit 2024-07-28T19:22:09.972923+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:11.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[31715]: audit 2024-07-28T19:22:10.327774+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:11.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[31715]: audit 2024-07-28T19:22:10.335430+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:11.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:10 smithi100 bash[31715]: audit 2024-07-28T19:22:10.345075+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:11.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: cluster 2024-07-28T19:22:09.359830+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-07-28T19:22:11.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: audit 2024-07-28T19:22:09.965324+0000 mon.a (mon.0) 414 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-07-28T19:22:11.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: cluster 2024-07-28T19:22:09.965458+0000 mon.a (mon.0) 415 : cluster [DBG] osdmap e25: 4 total, 3 up, 4 in 2024-07-28T19:22:11.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: audit 2024-07-28T19:22:09.965896+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:11.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: audit 2024-07-28T19:22:09.972923+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:11.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: audit 2024-07-28T19:22:10.327774+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:11.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: audit 2024-07-28T19:22:10.335430+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:11.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: audit 2024-07-28T19:22:10.345075+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:12.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[21828]: cluster 2024-07-28T19:22:09.540800+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[21828]: cluster 2024-07-28T19:22:09.540869+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[21828]: cluster 2024-07-28T19:22:10.965184+0000 mon.a (mon.0) 421 : cluster [INF] osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287] boot 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[21828]: cluster 2024-07-28T19:22:10.965270+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[21828]: audit 2024-07-28T19:22:10.965886+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[31715]: cluster 2024-07-28T19:22:09.540800+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[31715]: cluster 2024-07-28T19:22:09.540869+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[31715]: cluster 2024-07-28T19:22:10.965184+0000 mon.a (mon.0) 421 : cluster [INF] osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287] boot 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[31715]: cluster 2024-07-28T19:22:10.965270+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-07-28T19:22:12.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:11 smithi100 bash[31715]: audit 2024-07-28T19:22:10.965886+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:12.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: cluster 2024-07-28T19:22:09.540800+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:22:12.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: cluster 2024-07-28T19:22:09.540869+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:22:12.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: cluster 2024-07-28T19:22:10.965184+0000 mon.a (mon.0) 421 : cluster [INF] osd.3 [v2:172.21.15.100:6826/684533287,v1:172.21.15.100:6827/684533287] boot 2024-07-28T19:22:12.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: cluster 2024-07-28T19:22:10.965270+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-07-28T19:22:12.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:11 smithi138 bash[23952]: audit 2024-07-28T19:22:10.965886+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:22:13.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:12 smithi100 bash[21828]: cluster 2024-07-28T19:22:11.360337+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:13.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:12 smithi100 bash[21828]: cluster 2024-07-28T19:22:11.981833+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-07-28T19:22:13.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:12 smithi100 bash[31715]: cluster 2024-07-28T19:22:11.360337+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:13.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:12 smithi100 bash[31715]: cluster 2024-07-28T19:22:11.981833+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-07-28T19:22:13.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:12 smithi138 bash[23952]: cluster 2024-07-28T19:22:11.360337+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:13.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:12 smithi138 bash[23952]: cluster 2024-07-28T19:22:11.981833+0000 mon.a (mon.0) 424 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2024-07-28T19:22:15.232 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:14 smithi100 bash[21828]: cluster 2024-07-28T19:22:13.360862+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:15.232 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:14 smithi100 bash[31715]: cluster 2024-07-28T19:22:13.360862+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:15.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:14 smithi138 bash[23952]: cluster 2024-07-28T19:22:13.360862+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:15.659 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:22:17.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:16 smithi100 bash[21828]: cluster 2024-07-28T19:22:15.361382+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:17.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:16 smithi100 bash[31715]: cluster 2024-07-28T19:22:15.361382+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:17.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:16 smithi138 bash[23952]: cluster 2024-07-28T19:22:15.361382+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:18.639 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-07-28T19:22:18.798 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-07-28T19:22:18.815 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2024-07-28T19:22:18.816 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2024-07-28T19:22:18.816 INFO:teuthology.orchestra.run.smithi138.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0142901 s, 734 MB/s 2024-07-28T19:22:18.816 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2024-07-28T19:22:19.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:18 smithi100 bash[21828]: cluster 2024-07-28T19:22:17.361855+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:18 smithi100 bash[31715]: cluster 2024-07-28T19:22:17.361855+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:19.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:18 smithi138 bash[23952]: cluster 2024-07-28T19:22:17.361855+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:19.435 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch daemon add osd smithi138:vg_nvme/lv_4 2024-07-28T19:22:21.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: cluster 2024-07-28T19:22:19.362424+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:21.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: cephadm 2024-07-28T19:22:19.844623+0000 mgr.y (mgr.14150) 206 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:22:21.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.853212+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.861222+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.863389+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.864853+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.866141+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.867440+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: cephadm 2024-07-28T19:22:19.868405+0000 mgr.y (mgr.14150) 207 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 4060M 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.876065+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.878487+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.879775+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[21828]: audit 2024-07-28T19:22:19.887136+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: cluster 2024-07-28T19:22:19.362424+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: cephadm 2024-07-28T19:22:19.844623+0000 mgr.y (mgr.14150) 206 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.853212+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.861222+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.863389+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.864853+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.866141+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.867440+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: cephadm 2024-07-28T19:22:19.868405+0000 mgr.y (mgr.14150) 207 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 4060M 2024-07-28T19:22:21.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.876065+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.878487+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:21.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.879775+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:21.251 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:20 smithi100 bash[31715]: audit 2024-07-28T19:22:19.887136+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: cluster 2024-07-28T19:22:19.362424+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: cephadm 2024-07-28T19:22:19.844623+0000 mgr.y (mgr.14150) 206 : cephadm [INF] Detected new or changed devices on smithi100 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.853212+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.861222+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.863389+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.864853+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.866141+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.867440+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: cephadm 2024-07-28T19:22:19.868405+0000 mgr.y (mgr.14150) 207 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 4060M 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.876065+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:21.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.878487+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:21.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.879775+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:21.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:20 smithi138 bash[23952]: audit 2024-07-28T19:22:19.887136+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:23.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:22 smithi100 bash[21828]: cluster 2024-07-28T19:22:21.363084+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:22 smithi100 bash[31715]: cluster 2024-07-28T19:22:21.363084+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:23.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:22 smithi138 bash[23952]: cluster 2024-07-28T19:22:21.363084+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:24.148 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:22:25.124 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:24 smithi138 bash[23952]: cluster 2024-07-28T19:22:23.363697+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:25.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:24 smithi100 bash[21828]: cluster 2024-07-28T19:22:23.363697+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:25.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:24 smithi100 bash[31715]: cluster 2024-07-28T19:22:23.363697+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:27.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:26 smithi138 bash[23952]: cluster 2024-07-28T19:22:25.364287+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:27.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:26 smithi100 bash[21828]: cluster 2024-07-28T19:22:25.364287+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:27.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:26 smithi100 bash[31715]: cluster 2024-07-28T19:22:25.364287+0000 mgr.y (mgr.14150) 210 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:28.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:27 smithi100 bash[31715]: audit 2024-07-28T19:22:27.044046+0000 mgr.y (mgr.14150) 211 : audit [DBG] from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:22:28.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:27 smithi100 bash[31715]: audit 2024-07-28T19:22:27.050760+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:22:28.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:27 smithi100 bash[31715]: audit 2024-07-28T19:22:27.056931+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:22:28.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:27 smithi100 bash[31715]: audit 2024-07-28T19:22:27.058670+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:28.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:27 smithi100 bash[21828]: audit 2024-07-28T19:22:27.044046+0000 mgr.y (mgr.14150) 211 : audit [DBG] from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:22:28.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:27 smithi100 bash[21828]: audit 2024-07-28T19:22:27.050760+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:22:28.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:27 smithi100 bash[21828]: audit 2024-07-28T19:22:27.056931+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:22:28.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:27 smithi100 bash[21828]: audit 2024-07-28T19:22:27.058670+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:28.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:27 smithi138 bash[23952]: audit 2024-07-28T19:22:27.044046+0000 mgr.y (mgr.14150) 211 : audit [DBG] from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:22:28.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:27 smithi138 bash[23952]: audit 2024-07-28T19:22:27.050760+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:22:28.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:27 smithi138 bash[23952]: audit 2024-07-28T19:22:27.056931+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:22:28.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:27 smithi138 bash[23952]: audit 2024-07-28T19:22:27.058670+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:29.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:28 smithi100 bash[31715]: cluster 2024-07-28T19:22:27.364771+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:29.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:28 smithi100 bash[21828]: cluster 2024-07-28T19:22:27.364771+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:29.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:28 smithi138 bash[23952]: cluster 2024-07-28T19:22:27.364771+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:31.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:30 smithi100 bash[21828]: cluster 2024-07-28T19:22:29.365314+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:31.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:30 smithi100 bash[31715]: cluster 2024-07-28T19:22:29.365314+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:31.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:30 smithi138 bash[23952]: cluster 2024-07-28T19:22:29.365314+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:33.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:32 smithi138 bash[23952]: cluster 2024-07-28T19:22:31.365842+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:33.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:32 smithi100 bash[21828]: cluster 2024-07-28T19:22:31.365842+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:33.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:32 smithi100 bash[31715]: cluster 2024-07-28T19:22:31.365842+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:35.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:34 smithi100 bash[21828]: cluster 2024-07-28T19:22:33.366352+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:34 smithi100 bash[31715]: cluster 2024-07-28T19:22:33.366352+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:34 smithi138 bash[23952]: cluster 2024-07-28T19:22:33.366352+0000 mgr.y (mgr.14150) 215 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:36.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[21828]: audit 2024-07-28T19:22:35.228754+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.138:0/1041709783' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]: dispatch 2024-07-28T19:22:36.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[21828]: audit 2024-07-28T19:22:35.230057+0000 mon.a (mon.0) 438 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]: dispatch 2024-07-28T19:22:36.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[21828]: audit 2024-07-28T19:22:35.238148+0000 mon.a (mon.0) 439 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]': finished 2024-07-28T19:22:36.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[21828]: cluster 2024-07-28T19:22:35.238271+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-07-28T19:22:36.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[21828]: audit 2024-07-28T19:22:35.238487+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:22:36.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[21828]: audit 2024-07-28T19:22:35.857888+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.138:0/3993998231' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:22:36.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[31715]: audit 2024-07-28T19:22:35.228754+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.138:0/1041709783' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]: dispatch 2024-07-28T19:22:36.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[31715]: audit 2024-07-28T19:22:35.230057+0000 mon.a (mon.0) 438 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]: dispatch 2024-07-28T19:22:36.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[31715]: audit 2024-07-28T19:22:35.238148+0000 mon.a (mon.0) 439 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]': finished 2024-07-28T19:22:36.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[31715]: cluster 2024-07-28T19:22:35.238271+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-07-28T19:22:36.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[31715]: audit 2024-07-28T19:22:35.238487+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:22:36.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:35 smithi100 bash[31715]: audit 2024-07-28T19:22:35.857888+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.138:0/3993998231' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:22:36.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:35 smithi138 bash[23952]: audit 2024-07-28T19:22:35.228754+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.138:0/1041709783' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]: dispatch 2024-07-28T19:22:36.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:35 smithi138 bash[23952]: audit 2024-07-28T19:22:35.230057+0000 mon.a (mon.0) 438 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]: dispatch 2024-07-28T19:22:36.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:35 smithi138 bash[23952]: audit 2024-07-28T19:22:35.238148+0000 mon.a (mon.0) 439 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "75f51136-e1cc-4d3a-89d8-1618aaff366e"}]': finished 2024-07-28T19:22:36.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:35 smithi138 bash[23952]: cluster 2024-07-28T19:22:35.238271+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-07-28T19:22:36.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:35 smithi138 bash[23952]: audit 2024-07-28T19:22:35.238487+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:22:36.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:35 smithi138 bash[23952]: audit 2024-07-28T19:22:35.857888+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.138:0/3993998231' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:22:37.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:36 smithi100 bash[21828]: cluster 2024-07-28T19:22:35.366899+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:37.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:36 smithi100 bash[31715]: cluster 2024-07-28T19:22:35.366899+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:37.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:36 smithi138 bash[23952]: cluster 2024-07-28T19:22:35.366899+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:39.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:38 smithi100 bash[21828]: cluster 2024-07-28T19:22:37.367447+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:39.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:38 smithi100 bash[31715]: cluster 2024-07-28T19:22:37.367447+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:39.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:38 smithi138 bash[23952]: cluster 2024-07-28T19:22:37.367447+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:41.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:40 smithi100 bash[21828]: cluster 2024-07-28T19:22:39.367983+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:41.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:40 smithi100 bash[31715]: cluster 2024-07-28T19:22:39.367983+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:41.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:40 smithi138 bash[23952]: cluster 2024-07-28T19:22:39.367983+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:43.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:42 smithi100 bash[21828]: cluster 2024-07-28T19:22:41.368546+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:43.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:42 smithi100 bash[31715]: cluster 2024-07-28T19:22:41.368546+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:43.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:42 smithi138 bash[23952]: cluster 2024-07-28T19:22:41.368546+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:45.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:44 smithi138 bash[23952]: cluster 2024-07-28T19:22:43.369098+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:45.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:44 smithi100 bash[21828]: cluster 2024-07-28T19:22:43.369098+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:45.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:44 smithi100 bash[31715]: cluster 2024-07-28T19:22:43.369098+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:47.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:46 smithi138 bash[23952]: cluster 2024-07-28T19:22:45.369638+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:47.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:46 smithi100 bash[21828]: cluster 2024-07-28T19:22:45.369638+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:47.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:46 smithi100 bash[31715]: cluster 2024-07-28T19:22:45.369638+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:48.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:47 smithi100 bash[21828]: audit 2024-07-28T19:22:47.311595+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:22:48.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:47 smithi100 bash[21828]: audit 2024-07-28T19:22:47.313259+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:48.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:47 smithi100 bash[31715]: audit 2024-07-28T19:22:47.311595+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:22:48.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:47 smithi100 bash[31715]: audit 2024-07-28T19:22:47.313259+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:48.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:47 smithi138 bash[23952]: audit 2024-07-28T19:22:47.311595+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:22:48.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:47 smithi138 bash[23952]: audit 2024-07-28T19:22:47.313259+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:49.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:48 smithi100 bash[21828]: cephadm 2024-07-28T19:22:47.314687+0000 mgr.y (mgr.14150) 222 : cephadm [INF] Deploying daemon osd.4 on smithi138 2024-07-28T19:22:49.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:48 smithi100 bash[21828]: cluster 2024-07-28T19:22:47.370251+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:49.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:48 smithi100 bash[31715]: cephadm 2024-07-28T19:22:47.314687+0000 mgr.y (mgr.14150) 222 : cephadm [INF] Deploying daemon osd.4 on smithi138 2024-07-28T19:22:49.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:48 smithi100 bash[31715]: cluster 2024-07-28T19:22:47.370251+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:49.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:48 smithi138 bash[23952]: cephadm 2024-07-28T19:22:47.314687+0000 mgr.y (mgr.14150) 222 : cephadm [INF] Deploying daemon osd.4 on smithi138 2024-07-28T19:22:49.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:48 smithi138 bash[23952]: cluster 2024-07-28T19:22:47.370251+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:51.180 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:50 smithi138 bash[23952]: cluster 2024-07-28T19:22:49.370860+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:51.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:50 smithi100 bash[21828]: cluster 2024-07-28T19:22:49.370860+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:51.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:50 smithi100 bash[31715]: cluster 2024-07-28T19:22:49.370860+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:51.513 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:51 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:51.513 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:51 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:51.514 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:22:51 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:51.514 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:22:51 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:22:52.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:51 smithi100 bash[31715]: audit 2024-07-28T19:22:51.548238+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:52.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:51 smithi100 bash[31715]: audit 2024-07-28T19:22:51.556026+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:52.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:51 smithi100 bash[31715]: audit 2024-07-28T19:22:51.564221+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:52.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:51 smithi100 bash[21828]: audit 2024-07-28T19:22:51.548238+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:52.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:51 smithi100 bash[21828]: audit 2024-07-28T19:22:51.556026+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:52.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:51 smithi100 bash[21828]: audit 2024-07-28T19:22:51.564221+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:52.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:51 smithi138 bash[23952]: audit 2024-07-28T19:22:51.548238+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:22:52.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:51 smithi138 bash[23952]: audit 2024-07-28T19:22:51.556026+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:52.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:51 smithi138 bash[23952]: audit 2024-07-28T19:22:51.564221+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:53.234 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:52 smithi138 bash[23952]: cluster 2024-07-28T19:22:51.371457+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:52 smithi100 bash[31715]: cluster 2024-07-28T19:22:51.371457+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:53.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:52 smithi100 bash[21828]: cluster 2024-07-28T19:22:51.371457+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:55.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:54 smithi100 bash[21828]: cluster 2024-07-28T19:22:53.371951+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:55.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:54 smithi100 bash[31715]: cluster 2024-07-28T19:22:53.371951+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:55.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:54 smithi138 bash[23952]: cluster 2024-07-28T19:22:53.371951+0000 mgr.y (mgr.14150) 226 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:57.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:56 smithi100 bash[21828]: cluster 2024-07-28T19:22:55.372470+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:57.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:56 smithi100 bash[21828]: audit 2024-07-28T19:22:56.605648+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:57.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:56 smithi100 bash[21828]: audit 2024-07-28T19:22:56.613862+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:57.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:56 smithi100 bash[31715]: cluster 2024-07-28T19:22:55.372470+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:57.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:56 smithi100 bash[31715]: audit 2024-07-28T19:22:56.605648+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:57.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:56 smithi100 bash[31715]: audit 2024-07-28T19:22:56.613862+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:57.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:56 smithi138 bash[23952]: cluster 2024-07-28T19:22:55.372470+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:57.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:56 smithi138 bash[23952]: audit 2024-07-28T19:22:56.605648+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:57.302 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:56 smithi138 bash[23952]: audit 2024-07-28T19:22:56.613862+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:58.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:57 smithi100 bash[21828]: audit 2024-07-28T19:22:57.056852+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:58.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:57 smithi100 bash[21828]: audit 2024-07-28T19:22:57.058417+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:58.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:57 smithi100 bash[21828]: audit 2024-07-28T19:22:57.067963+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:58.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:57 smithi100 bash[31715]: audit 2024-07-28T19:22:57.056852+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:58.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:57 smithi100 bash[31715]: audit 2024-07-28T19:22:57.058417+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:58.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:57 smithi100 bash[31715]: audit 2024-07-28T19:22:57.067963+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:58.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:57 smithi138 bash[23952]: audit 2024-07-28T19:22:57.056852+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:22:58.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:57 smithi138 bash[23952]: audit 2024-07-28T19:22:57.058417+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:22:58.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:57 smithi138 bash[23952]: audit 2024-07-28T19:22:57.067963+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:22:59.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:59 smithi138 bash[23952]: cluster 2024-07-28T19:22:57.373007+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:59.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:59 smithi138 bash[23952]: audit 2024-07-28T19:22:58.177133+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:22:59.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:22:59 smithi138 bash[23952]: audit 2024-07-28T19:22:58.178369+0000 mon.a (mon.0) 452 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:22:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:59 smithi100 bash[21828]: cluster 2024-07-28T19:22:57.373007+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:59 smithi100 bash[21828]: audit 2024-07-28T19:22:58.177133+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:22:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:22:59 smithi100 bash[21828]: audit 2024-07-28T19:22:58.178369+0000 mon.a (mon.0) 452 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:22:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:59 smithi100 bash[31715]: cluster 2024-07-28T19:22:57.373007+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:22:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:59 smithi100 bash[31715]: audit 2024-07-28T19:22:58.177133+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:22:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:22:59 smithi100 bash[31715]: audit 2024-07-28T19:22:58.178369+0000 mon.a (mon.0) 452 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:22:59.983 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 4 on host 'smithi138' 2024-07-28T19:23:00.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:00 smithi138 bash[23952]: audit 2024-07-28T19:22:59.080406+0000 mon.a (mon.0) 453 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-28T19:23:00.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:00 smithi138 bash[23952]: cluster 2024-07-28T19:22:59.080608+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-07-28T19:23:00.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:00 smithi138 bash[23952]: audit 2024-07-28T19:22:59.080886+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:00.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:00 smithi138 bash[23952]: audit 2024-07-28T19:22:59.081174+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:00.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:00 smithi138 bash[23952]: audit 2024-07-28T19:22:59.082438+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:00.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:00 smithi138 bash[23952]: audit 2024-07-28T19:22:59.961697+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:00.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:00 smithi138 bash[23952]: audit 2024-07-28T19:22:59.971192+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:00.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:00 smithi138 bash[23952]: audit 2024-07-28T19:22:59.981232+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:00.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[21828]: audit 2024-07-28T19:22:59.080406+0000 mon.a (mon.0) 453 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-28T19:23:00.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[21828]: cluster 2024-07-28T19:22:59.080608+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-07-28T19:23:00.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[21828]: audit 2024-07-28T19:22:59.080886+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:00.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[21828]: audit 2024-07-28T19:22:59.081174+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:00.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[21828]: audit 2024-07-28T19:22:59.082438+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:00.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[21828]: audit 2024-07-28T19:22:59.961697+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:00.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[21828]: audit 2024-07-28T19:22:59.971192+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[21828]: audit 2024-07-28T19:22:59.981232+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[31715]: audit 2024-07-28T19:22:59.080406+0000 mon.a (mon.0) 453 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[31715]: cluster 2024-07-28T19:22:59.080608+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[31715]: audit 2024-07-28T19:22:59.080886+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[31715]: audit 2024-07-28T19:22:59.081174+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[31715]: audit 2024-07-28T19:22:59.082438+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[31715]: audit 2024-07-28T19:22:59.961697+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[31715]: audit 2024-07-28T19:22:59.971192+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:00.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:00 smithi100 bash[31715]: audit 2024-07-28T19:22:59.981232+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:00.579 DEBUG:teuthology.orchestra.run.smithi138:osd.4> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.4.service 2024-07-28T19:23:00.581 INFO:tasks.cephadm:Deploying osd.5 on smithi138 with /dev/vg_nvme/lv_3... 2024-07-28T19:23:00.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-07-28T19:23:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[21828]: cluster 2024-07-28T19:22:59.373453+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:23:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[21828]: audit 2024-07-28T19:23:00.089137+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:23:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[21828]: cluster 2024-07-28T19:23:00.089254+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-07-28T19:23:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[21828]: audit 2024-07-28T19:23:00.089701+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[21828]: audit 2024-07-28T19:23:00.096499+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:01.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[31715]: cluster 2024-07-28T19:22:59.373453+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:23:01.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[31715]: audit 2024-07-28T19:23:00.089137+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:23:01.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[31715]: cluster 2024-07-28T19:23:00.089254+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-07-28T19:23:01.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[31715]: audit 2024-07-28T19:23:00.089701+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:01.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:01 smithi100 bash[31715]: audit 2024-07-28T19:23:00.096499+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:01.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:01 smithi138 bash[23952]: cluster 2024-07-28T19:22:59.373453+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:23:01.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:01 smithi138 bash[23952]: audit 2024-07-28T19:23:00.089137+0000 mon.a (mon.0) 460 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:23:01.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:01 smithi138 bash[23952]: cluster 2024-07-28T19:23:00.089254+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e30: 5 total, 4 up, 5 in 2024-07-28T19:23:01.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:01 smithi138 bash[23952]: audit 2024-07-28T19:23:00.089701+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:01.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:01 smithi138 bash[23952]: audit 2024-07-28T19:23:00.096499+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:02.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[21828]: cluster 2024-07-28T19:22:59.198727+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:23:02.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[21828]: cluster 2024-07-28T19:22:59.198786+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:23:02.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[21828]: cluster 2024-07-28T19:23:01.099254+0000 mon.a (mon.0) 464 : cluster [INF] osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242] boot 2024-07-28T19:23:02.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[21828]: cluster 2024-07-28T19:23:01.099327+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-07-28T19:23:02.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[21828]: audit 2024-07-28T19:23:01.099891+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:02.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[31715]: cluster 2024-07-28T19:22:59.198727+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:23:02.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[31715]: cluster 2024-07-28T19:22:59.198786+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:23:02.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[31715]: cluster 2024-07-28T19:23:01.099254+0000 mon.a (mon.0) 464 : cluster [INF] osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242] boot 2024-07-28T19:23:02.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[31715]: cluster 2024-07-28T19:23:01.099327+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-07-28T19:23:02.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:02 smithi100 bash[31715]: audit 2024-07-28T19:23:01.099891+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:02.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:02 smithi138 bash[23952]: cluster 2024-07-28T19:22:59.198727+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:23:02.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:02 smithi138 bash[23952]: cluster 2024-07-28T19:22:59.198786+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:23:02.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:02 smithi138 bash[23952]: cluster 2024-07-28T19:23:01.099254+0000 mon.a (mon.0) 464 : cluster [INF] osd.4 [v2:172.21.15.138:6800/1306402242,v1:172.21.15.138:6801/1306402242] boot 2024-07-28T19:23:02.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:02 smithi138 bash[23952]: cluster 2024-07-28T19:23:01.099327+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-07-28T19:23:02.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:02 smithi138 bash[23952]: audit 2024-07-28T19:23:01.099891+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:23:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:03 smithi100 bash[21828]: cluster 2024-07-28T19:23:01.374050+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:23:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:03 smithi100 bash[21828]: cluster 2024-07-28T19:23:02.110290+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-07-28T19:23:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:03 smithi100 bash[31715]: cluster 2024-07-28T19:23:01.374050+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:23:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:03 smithi100 bash[31715]: cluster 2024-07-28T19:23:02.110290+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-07-28T19:23:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:03 smithi138 bash[23952]: cluster 2024-07-28T19:23:01.374050+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-07-28T19:23:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:03 smithi138 bash[23952]: cluster 2024-07-28T19:23:02.110290+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-07-28T19:23:04.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:04 smithi100 bash[21828]: cluster 2024-07-28T19:23:03.113207+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-07-28T19:23:04.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:04 smithi100 bash[31715]: cluster 2024-07-28T19:23:03.113207+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-07-28T19:23:04.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:04 smithi138 bash[23952]: cluster 2024-07-28T19:23:03.113207+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e33: 5 total, 5 up, 5 in 2024-07-28T19:23:04.724 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:23:05.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:05 smithi100 bash[21828]: cluster 2024-07-28T19:23:03.374613+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:05.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:05 smithi100 bash[21828]: cluster 2024-07-28T19:23:04.131458+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2024-07-28T19:23:05.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:05 smithi100 bash[31715]: cluster 2024-07-28T19:23:03.374613+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:05.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:05 smithi100 bash[31715]: cluster 2024-07-28T19:23:04.131458+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2024-07-28T19:23:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:05 smithi138 bash[23952]: cluster 2024-07-28T19:23:03.374613+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:05 smithi138 bash[23952]: cluster 2024-07-28T19:23:04.131458+0000 mon.a (mon.0) 469 : cluster [DBG] osdmap e34: 5 total, 5 up, 5 in 2024-07-28T19:23:06.393 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:06 smithi138 bash[23952]: cluster 2024-07-28T19:23:05.133541+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e35: 5 total, 5 up, 5 in 2024-07-28T19:23:06.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:06 smithi100 bash[21828]: cluster 2024-07-28T19:23:05.133541+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e35: 5 total, 5 up, 5 in 2024-07-28T19:23:06.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:06 smithi100 bash[31715]: cluster 2024-07-28T19:23:05.133541+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e35: 5 total, 5 up, 5 in 2024-07-28T19:23:07.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:07 smithi138 bash[23952]: cluster 2024-07-28T19:23:05.375198+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v208: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-07-28T19:23:07.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:07 smithi100 bash[21828]: cluster 2024-07-28T19:23:05.375198+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v208: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-07-28T19:23:07.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:07 smithi100 bash[31715]: cluster 2024-07-28T19:23:05.375198+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v208: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-07-28T19:23:09.021 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-07-28T19:23:09.159 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-07-28T19:23:09.174 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2024-07-28T19:23:09.174 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2024-07-28T19:23:09.174 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133755 s, 784 MB/s 2024-07-28T19:23:09.174 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2024-07-28T19:23:09.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:09 smithi100 bash[21828]: cluster 2024-07-28T19:23:07.375749+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v209: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-07-28T19:23:09.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:09 smithi100 bash[31715]: cluster 2024-07-28T19:23:07.375749+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v209: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-07-28T19:23:09.519 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:09 smithi138 bash[23952]: cluster 2024-07-28T19:23:07.375749+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v209: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-07-28T19:23:09.926 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch daemon add osd smithi138:vg_nvme/lv_3 2024-07-28T19:23:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[21828]: cluster 2024-07-28T19:23:09.376330+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v210: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-07-28T19:23:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[21828]: audit 2024-07-28T19:23:10.274448+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[21828]: audit 2024-07-28T19:23:10.283013+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[21828]: audit 2024-07-28T19:23:10.284815+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:23:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[21828]: audit 2024-07-28T19:23:10.294588+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[21828]: audit 2024-07-28T19:23:10.296914+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[21828]: audit 2024-07-28T19:23:10.298561+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[21828]: audit 2024-07-28T19:23:10.308843+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[31715]: cluster 2024-07-28T19:23:09.376330+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v210: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[31715]: audit 2024-07-28T19:23:10.274448+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[31715]: audit 2024-07-28T19:23:10.283013+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[31715]: audit 2024-07-28T19:23:10.284815+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[31715]: audit 2024-07-28T19:23:10.294588+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[31715]: audit 2024-07-28T19:23:10.296914+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[31715]: audit 2024-07-28T19:23:10.298561+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:23:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:11 smithi100 bash[31715]: audit 2024-07-28T19:23:10.308843+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:11 smithi138 bash[23952]: cluster 2024-07-28T19:23:09.376330+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v210: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-07-28T19:23:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:11 smithi138 bash[23952]: audit 2024-07-28T19:23:10.274448+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:11 smithi138 bash[23952]: audit 2024-07-28T19:23:10.283013+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:11 smithi138 bash[23952]: audit 2024-07-28T19:23:10.284815+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:23:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:11 smithi138 bash[23952]: audit 2024-07-28T19:23:10.294588+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:11 smithi138 bash[23952]: audit 2024-07-28T19:23:10.296914+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:11 smithi138 bash[23952]: audit 2024-07-28T19:23:10.298561+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:23:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:11 smithi138 bash[23952]: audit 2024-07-28T19:23:10.308843+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:12.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:12 smithi100 bash[21828]: cephadm 2024-07-28T19:23:10.264481+0000 mgr.y (mgr.14150) 235 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:23:12.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:12 smithi100 bash[21828]: cephadm 2024-07-28T19:23:10.286137+0000 mgr.y (mgr.14150) 236 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 17265M 2024-07-28T19:23:12.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:12 smithi100 bash[31715]: cephadm 2024-07-28T19:23:10.264481+0000 mgr.y (mgr.14150) 235 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:23:12.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:12 smithi100 bash[31715]: cephadm 2024-07-28T19:23:10.286137+0000 mgr.y (mgr.14150) 236 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 17265M 2024-07-28T19:23:12.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:12 smithi138 bash[23952]: cephadm 2024-07-28T19:23:10.264481+0000 mgr.y (mgr.14150) 235 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:23:12.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:12 smithi138 bash[23952]: cephadm 2024-07-28T19:23:10.286137+0000 mgr.y (mgr.14150) 236 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 17265M 2024-07-28T19:23:13.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:13 smithi100 bash[21828]: cluster 2024-07-28T19:23:11.376918+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-28T19:23:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:13 smithi100 bash[31715]: cluster 2024-07-28T19:23:11.376918+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-28T19:23:13.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:13 smithi138 bash[23952]: cluster 2024-07-28T19:23:11.376918+0000 mgr.y (mgr.14150) 237 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-28T19:23:14.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:14 smithi100 bash[21828]: cluster 2024-07-28T19:23:13.377448+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-07-28T19:23:14.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:14 smithi100 bash[31715]: cluster 2024-07-28T19:23:13.377448+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-07-28T19:23:14.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:14 smithi138 bash[23952]: cluster 2024-07-28T19:23:13.377448+0000 mgr.y (mgr.14150) 238 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-07-28T19:23:14.646 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:23:16.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:16 smithi100 bash[21828]: cluster 2024-07-28T19:23:15.377990+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:16.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:16 smithi100 bash[31715]: cluster 2024-07-28T19:23:15.377990+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:16 smithi138 bash[23952]: cluster 2024-07-28T19:23:15.377990+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:18.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[21828]: cluster 2024-07-28T19:23:17.378510+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[21828]: audit 2024-07-28T19:23:17.480281+0000 mgr.y (mgr.14150) 241 : audit [DBG] from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[21828]: audit 2024-07-28T19:23:17.486637+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[21828]: audit 2024-07-28T19:23:17.492519+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[21828]: audit 2024-07-28T19:23:17.494154+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[31715]: cluster 2024-07-28T19:23:17.378510+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[31715]: audit 2024-07-28T19:23:17.480281+0000 mgr.y (mgr.14150) 241 : audit [DBG] from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[31715]: audit 2024-07-28T19:23:17.486637+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[31715]: audit 2024-07-28T19:23:17.492519+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:23:18.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:18 smithi100 bash[31715]: audit 2024-07-28T19:23:17.494154+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:18.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:18 smithi138 bash[23952]: cluster 2024-07-28T19:23:17.378510+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:18.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:18 smithi138 bash[23952]: audit 2024-07-28T19:23:17.480281+0000 mgr.y (mgr.14150) 241 : audit [DBG] from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:23:18.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:18 smithi138 bash[23952]: audit 2024-07-28T19:23:17.486637+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:23:18.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:18 smithi138 bash[23952]: audit 2024-07-28T19:23:17.492519+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:23:18.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:18 smithi138 bash[23952]: audit 2024-07-28T19:23:17.494154+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:20.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:20 smithi100 bash[21828]: cluster 2024-07-28T19:23:19.379010+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:20.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:20 smithi100 bash[31715]: cluster 2024-07-28T19:23:19.379010+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:20.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:20 smithi138 bash[23952]: cluster 2024-07-28T19:23:19.379010+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:22.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:22 smithi100 bash[21828]: cluster 2024-07-28T19:23:21.379537+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:22.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:22 smithi100 bash[31715]: cluster 2024-07-28T19:23:21.379537+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:22.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:22 smithi138 bash[23952]: cluster 2024-07-28T19:23:21.379537+0000 mgr.y (mgr.14150) 243 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:24.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:24 smithi100 bash[21828]: cluster 2024-07-28T19:23:23.380042+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:24.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:24 smithi100 bash[31715]: cluster 2024-07-28T19:23:23.380042+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:24.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:24 smithi138 bash[23952]: cluster 2024-07-28T19:23:23.380042+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[21828]: cluster 2024-07-28T19:23:25.380547+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[21828]: audit 2024-07-28T19:23:25.533111+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.138:0/3291174074' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]: dispatch 2024-07-28T19:23:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[21828]: audit 2024-07-28T19:23:25.534656+0000 mon.a (mon.0) 481 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]: dispatch 2024-07-28T19:23:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[21828]: audit 2024-07-28T19:23:25.548800+0000 mon.a (mon.0) 482 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]': finished 2024-07-28T19:23:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[21828]: cluster 2024-07-28T19:23:25.548962+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-07-28T19:23:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[21828]: audit 2024-07-28T19:23:25.549275+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[21828]: audit 2024-07-28T19:23:26.194163+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.138:0/597355636' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:23:26.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[31715]: cluster 2024-07-28T19:23:25.380547+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:26.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[31715]: audit 2024-07-28T19:23:25.533111+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.138:0/3291174074' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]: dispatch 2024-07-28T19:23:26.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[31715]: audit 2024-07-28T19:23:25.534656+0000 mon.a (mon.0) 481 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]: dispatch 2024-07-28T19:23:26.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[31715]: audit 2024-07-28T19:23:25.548800+0000 mon.a (mon.0) 482 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]': finished 2024-07-28T19:23:26.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[31715]: cluster 2024-07-28T19:23:25.548962+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-07-28T19:23:26.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[31715]: audit 2024-07-28T19:23:25.549275+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:26.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:26 smithi100 bash[31715]: audit 2024-07-28T19:23:26.194163+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.138:0/597355636' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:23:26.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:26 smithi138 bash[23952]: cluster 2024-07-28T19:23:25.380547+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:26.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:26 smithi138 bash[23952]: audit 2024-07-28T19:23:25.533111+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.138:0/3291174074' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]: dispatch 2024-07-28T19:23:26.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:26 smithi138 bash[23952]: audit 2024-07-28T19:23:25.534656+0000 mon.a (mon.0) 481 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]: dispatch 2024-07-28T19:23:26.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:26 smithi138 bash[23952]: audit 2024-07-28T19:23:25.548800+0000 mon.a (mon.0) 482 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a3e4950-a201-4105-9e45-ef45ce2d5f3b"}]': finished 2024-07-28T19:23:26.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:26 smithi138 bash[23952]: cluster 2024-07-28T19:23:25.548962+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e36: 6 total, 5 up, 6 in 2024-07-28T19:23:26.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:26 smithi138 bash[23952]: audit 2024-07-28T19:23:25.549275+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:26.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:26 smithi138 bash[23952]: audit 2024-07-28T19:23:26.194163+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.138:0/597355636' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:23:28.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:28 smithi100 bash[21828]: cluster 2024-07-28T19:23:27.381094+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:28.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:28 smithi100 bash[31715]: cluster 2024-07-28T19:23:27.381094+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:28.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:28 smithi138 bash[23952]: cluster 2024-07-28T19:23:27.381094+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:30.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:30 smithi100 bash[21828]: cluster 2024-07-28T19:23:29.381605+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:30.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:30 smithi100 bash[31715]: cluster 2024-07-28T19:23:29.381605+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:30 smithi138 bash[23952]: cluster 2024-07-28T19:23:29.381605+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:32.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:32 smithi100 bash[21828]: cluster 2024-07-28T19:23:31.382167+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:32.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:32 smithi100 bash[31715]: cluster 2024-07-28T19:23:31.382167+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:32.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:32 smithi138 bash[23952]: cluster 2024-07-28T19:23:31.382167+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:34.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:34 smithi138 bash[23952]: cluster 2024-07-28T19:23:33.382703+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:34.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:34 smithi100 bash[21828]: cluster 2024-07-28T19:23:33.382703+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:34.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:34 smithi100 bash[31715]: cluster 2024-07-28T19:23:33.382703+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:36 smithi100 bash[21828]: cluster 2024-07-28T19:23:35.383211+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:36.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:36 smithi100 bash[31715]: cluster 2024-07-28T19:23:35.383211+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:36.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:36 smithi138 bash[23952]: cluster 2024-07-28T19:23:35.383211+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:38.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:38 smithi100 bash[21828]: cluster 2024-07-28T19:23:37.383726+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:38.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:38 smithi100 bash[21828]: audit 2024-07-28T19:23:37.765017+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-28T19:23:38.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:38 smithi100 bash[21828]: audit 2024-07-28T19:23:37.766845+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:38.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:38 smithi100 bash[21828]: cephadm 2024-07-28T19:23:37.768314+0000 mgr.y (mgr.14150) 252 : cephadm [INF] Deploying daemon osd.5 on smithi138 2024-07-28T19:23:38.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:38 smithi100 bash[31715]: cluster 2024-07-28T19:23:37.383726+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:38.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:38 smithi100 bash[31715]: audit 2024-07-28T19:23:37.765017+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-28T19:23:38.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:38 smithi100 bash[31715]: audit 2024-07-28T19:23:37.766845+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:38.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:38 smithi100 bash[31715]: cephadm 2024-07-28T19:23:37.768314+0000 mgr.y (mgr.14150) 252 : cephadm [INF] Deploying daemon osd.5 on smithi138 2024-07-28T19:23:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:38 smithi138 bash[23952]: cluster 2024-07-28T19:23:37.383726+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:38 smithi138 bash[23952]: audit 2024-07-28T19:23:37.765017+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-28T19:23:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:38 smithi138 bash[23952]: audit 2024-07-28T19:23:37.766845+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:38 smithi138 bash[23952]: cephadm 2024-07-28T19:23:37.768314+0000 mgr.y (mgr.14150) 252 : cephadm [INF] Deploying daemon osd.5 on smithi138 2024-07-28T19:23:40.734 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:40 smithi138 bash[23952]: cluster 2024-07-28T19:23:39.384227+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:40.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:40 smithi100 bash[21828]: cluster 2024-07-28T19:23:39.384227+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:40.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:40 smithi100 bash[31715]: cluster 2024-07-28T19:23:39.384227+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:41.800 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:23:41.800 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:23:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:23:41.801 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:23:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:23:42.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:23:42.057 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:23:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:23:42.057 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:23:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:23:42.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:42 smithi100 bash[21828]: cluster 2024-07-28T19:23:41.384774+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:42.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:42 smithi100 bash[21828]: audit 2024-07-28T19:23:41.940672+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:42.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:42 smithi100 bash[21828]: audit 2024-07-28T19:23:41.948060+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:42.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:42 smithi100 bash[21828]: audit 2024-07-28T19:23:41.956269+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:42.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:42 smithi100 bash[31715]: cluster 2024-07-28T19:23:41.384774+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:42.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:42 smithi100 bash[31715]: audit 2024-07-28T19:23:41.940672+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:42.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:42 smithi100 bash[31715]: audit 2024-07-28T19:23:41.948060+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:42.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:42 smithi100 bash[31715]: audit 2024-07-28T19:23:41.956269+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:42 smithi138 bash[23952]: cluster 2024-07-28T19:23:41.384774+0000 mgr.y (mgr.14150) 254 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:42 smithi138 bash[23952]: audit 2024-07-28T19:23:41.940672+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:42 smithi138 bash[23952]: audit 2024-07-28T19:23:41.948060+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:42.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:42 smithi138 bash[23952]: audit 2024-07-28T19:23:41.956269+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:44.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:44 smithi100 bash[21828]: cluster 2024-07-28T19:23:43.385325+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:44.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:44 smithi100 bash[31715]: cluster 2024-07-28T19:23:43.385325+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:44.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:44 smithi138 bash[23952]: cluster 2024-07-28T19:23:43.385325+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:46.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:46 smithi100 bash[21828]: cluster 2024-07-28T19:23:45.385827+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:46.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:46 smithi100 bash[31715]: cluster 2024-07-28T19:23:45.385827+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:46.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:46 smithi138 bash[23952]: cluster 2024-07-28T19:23:45.385827+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:48.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[31715]: audit 2024-07-28T19:23:46.982473+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:48.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[31715]: audit 2024-07-28T19:23:46.991052+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:48.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[31715]: audit 2024-07-28T19:23:46.992787+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:48.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[31715]: audit 2024-07-28T19:23:46.994830+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:23:48.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[31715]: audit 2024-07-28T19:23:47.004646+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:48.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[21828]: audit 2024-07-28T19:23:46.982473+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:48.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[21828]: audit 2024-07-28T19:23:46.991052+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:48.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[21828]: audit 2024-07-28T19:23:46.992787+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:48.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[21828]: audit 2024-07-28T19:23:46.994830+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:23:48.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:47 smithi100 bash[21828]: audit 2024-07-28T19:23:47.004646+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:48.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:47 smithi138 bash[23952]: audit 2024-07-28T19:23:46.982473+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:48.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:47 smithi138 bash[23952]: audit 2024-07-28T19:23:46.991052+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:48.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:47 smithi138 bash[23952]: audit 2024-07-28T19:23:46.992787+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:23:48.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:47 smithi138 bash[23952]: audit 2024-07-28T19:23:46.994830+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:23:48.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:47 smithi138 bash[23952]: audit 2024-07-28T19:23:47.004646+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:49.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:49 smithi138 bash[23952]: cluster 2024-07-28T19:23:47.386364+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:49.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:49 smithi138 bash[23952]: audit 2024-07-28T19:23:48.163823+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-28T19:23:49.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:49 smithi138 bash[23952]: audit 2024-07-28T19:23:48.165410+0000 mon.a (mon.0) 495 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-28T19:23:49.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:49 smithi100 bash[21828]: cluster 2024-07-28T19:23:47.386364+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:49.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:49 smithi100 bash[21828]: audit 2024-07-28T19:23:48.163823+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-28T19:23:49.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:49 smithi100 bash[21828]: audit 2024-07-28T19:23:48.165410+0000 mon.a (mon.0) 495 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-28T19:23:49.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:49 smithi100 bash[31715]: cluster 2024-07-28T19:23:47.386364+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:49.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:49 smithi100 bash[31715]: audit 2024-07-28T19:23:48.163823+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-28T19:23:49.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:49 smithi100 bash[31715]: audit 2024-07-28T19:23:48.165410+0000 mon.a (mon.0) 495 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-07-28T19:23:50.053 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:50 smithi138 bash[23952]: audit 2024-07-28T19:23:49.020634+0000 mon.a (mon.0) 496 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-07-28T19:23:50.053 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:50 smithi138 bash[23952]: cluster 2024-07-28T19:23:49.020868+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2024-07-28T19:23:50.053 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:50 smithi138 bash[23952]: audit 2024-07-28T19:23:49.021167+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:50.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:50 smithi138 bash[23952]: audit 2024-07-28T19:23:49.021219+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:50.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:50 smithi138 bash[23952]: audit 2024-07-28T19:23:49.022757+0000 mon.a (mon.0) 499 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:50.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[31715]: audit 2024-07-28T19:23:49.020634+0000 mon.a (mon.0) 496 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-07-28T19:23:50.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[31715]: cluster 2024-07-28T19:23:49.020868+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2024-07-28T19:23:50.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[31715]: audit 2024-07-28T19:23:49.021167+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:50.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[31715]: audit 2024-07-28T19:23:49.021219+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:50.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[31715]: audit 2024-07-28T19:23:49.022757+0000 mon.a (mon.0) 499 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[21828]: audit 2024-07-28T19:23:49.020634+0000 mon.a (mon.0) 496 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-07-28T19:23:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[21828]: cluster 2024-07-28T19:23:49.020868+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e37: 6 total, 5 up, 6 in 2024-07-28T19:23:50.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[21828]: audit 2024-07-28T19:23:49.021167+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:50.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[21828]: audit 2024-07-28T19:23:49.021219+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:50.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:50 smithi100 bash[21828]: audit 2024-07-28T19:23:49.022757+0000 mon.a (mon.0) 499 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:23:50.552 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 5 on host 'smithi138' 2024-07-28T19:23:51.168 DEBUG:teuthology.orchestra.run.smithi138:osd.5> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.5.service 2024-07-28T19:23:51.171 INFO:tasks.cephadm:Deploying osd.6 on smithi138 with /dev/vg_nvme/lv_2... 2024-07-28T19:23:51.171 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-07-28T19:23:51.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:51 smithi138 bash[23952]: cluster 2024-07-28T19:23:49.386859+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:51.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:51 smithi138 bash[23952]: audit 2024-07-28T19:23:50.032018+0000 mon.a (mon.0) 500 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:23:51.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:51 smithi138 bash[23952]: cluster 2024-07-28T19:23:50.032239+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e38: 6 total, 5 up, 6 in 2024-07-28T19:23:51.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:51 smithi138 bash[23952]: audit 2024-07-28T19:23:50.032840+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:51.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:51 smithi138 bash[23952]: audit 2024-07-28T19:23:50.041197+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:51.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:51 smithi138 bash[23952]: audit 2024-07-28T19:23:50.534974+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:51.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:51 smithi138 bash[23952]: audit 2024-07-28T19:23:50.542455+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:51.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:51 smithi138 bash[23952]: audit 2024-07-28T19:23:50.550968+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:51.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[21828]: cluster 2024-07-28T19:23:49.386859+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:51.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[21828]: audit 2024-07-28T19:23:50.032018+0000 mon.a (mon.0) 500 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:23:51.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[21828]: cluster 2024-07-28T19:23:50.032239+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e38: 6 total, 5 up, 6 in 2024-07-28T19:23:51.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[21828]: audit 2024-07-28T19:23:50.032840+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:51.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[21828]: audit 2024-07-28T19:23:50.041197+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[21828]: audit 2024-07-28T19:23:50.534974+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[21828]: audit 2024-07-28T19:23:50.542455+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[21828]: audit 2024-07-28T19:23:50.550968+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[31715]: cluster 2024-07-28T19:23:49.386859+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[31715]: audit 2024-07-28T19:23:50.032018+0000 mon.a (mon.0) 500 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[31715]: cluster 2024-07-28T19:23:50.032239+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e38: 6 total, 5 up, 6 in 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[31715]: audit 2024-07-28T19:23:50.032840+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[31715]: audit 2024-07-28T19:23:50.041197+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[31715]: audit 2024-07-28T19:23:50.534974+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[31715]: audit 2024-07-28T19:23:50.542455+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:51.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:51 smithi100 bash[31715]: audit 2024-07-28T19:23:50.550968+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:23:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:52 smithi138 bash[23952]: cluster 2024-07-28T19:23:49.135235+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:23:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:52 smithi138 bash[23952]: cluster 2024-07-28T19:23:49.135319+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:23:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:52 smithi138 bash[23952]: cluster 2024-07-28T19:23:51.036897+0000 mon.a (mon.0) 507 : cluster [INF] osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487] boot 2024-07-28T19:23:52.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:52 smithi138 bash[23952]: cluster 2024-07-28T19:23:51.037007+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-07-28T19:23:52.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:52 smithi138 bash[23952]: audit 2024-07-28T19:23:51.037815+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:52.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[21828]: cluster 2024-07-28T19:23:49.135235+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:23:52.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[21828]: cluster 2024-07-28T19:23:49.135319+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:23:52.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[21828]: cluster 2024-07-28T19:23:51.036897+0000 mon.a (mon.0) 507 : cluster [INF] osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487] boot 2024-07-28T19:23:52.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[21828]: cluster 2024-07-28T19:23:51.037007+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-07-28T19:23:52.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[21828]: audit 2024-07-28T19:23:51.037815+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:52.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[31715]: cluster 2024-07-28T19:23:49.135235+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:23:52.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[31715]: cluster 2024-07-28T19:23:49.135319+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:23:52.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[31715]: cluster 2024-07-28T19:23:51.036897+0000 mon.a (mon.0) 507 : cluster [INF] osd.5 [v2:172.21.15.138:6808/1102784487,v1:172.21.15.138:6809/1102784487] boot 2024-07-28T19:23:52.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[31715]: cluster 2024-07-28T19:23:51.037007+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e39: 6 total, 6 up, 6 in 2024-07-28T19:23:52.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:52 smithi100 bash[31715]: audit 2024-07-28T19:23:51.037815+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:23:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:53 smithi138 bash[23952]: cluster 2024-07-28T19:23:51.387398+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:53 smithi138 bash[23952]: cluster 2024-07-28T19:23:52.057788+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2024-07-28T19:23:53.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:53 smithi100 bash[21828]: cluster 2024-07-28T19:23:51.387398+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:53.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:53 smithi100 bash[21828]: cluster 2024-07-28T19:23:52.057788+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2024-07-28T19:23:53.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:53 smithi100 bash[31715]: cluster 2024-07-28T19:23:51.387398+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:53.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:53 smithi100 bash[31715]: cluster 2024-07-28T19:23:52.057788+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e40: 6 total, 6 up, 6 in 2024-07-28T19:23:54.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:54 smithi100 bash[21828]: cluster 2024-07-28T19:23:53.057164+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2024-07-28T19:23:54.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:54 smithi100 bash[31715]: cluster 2024-07-28T19:23:53.057164+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2024-07-28T19:23:54.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:54 smithi138 bash[23952]: cluster 2024-07-28T19:23:53.057164+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e41: 6 total, 6 up, 6 in 2024-07-28T19:23:55.309 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:23:55.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:55 smithi138 bash[23952]: cluster 2024-07-28T19:23:53.388048+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v238: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:55.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:55 smithi138 bash[23952]: cluster 2024-07-28T19:23:54.058924+0000 mon.a (mon.0) 512 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:23:55.464 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:55 smithi138 bash[23952]: cluster 2024-07-28T19:23:54.072995+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2024-07-28T19:23:55.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:55 smithi100 bash[21828]: cluster 2024-07-28T19:23:53.388048+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v238: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:55.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:55 smithi100 bash[21828]: cluster 2024-07-28T19:23:54.058924+0000 mon.a (mon.0) 512 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:23:55.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:55 smithi100 bash[21828]: cluster 2024-07-28T19:23:54.072995+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2024-07-28T19:23:55.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:55 smithi100 bash[31715]: cluster 2024-07-28T19:23:53.388048+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v238: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:55.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:55 smithi100 bash[31715]: cluster 2024-07-28T19:23:54.058924+0000 mon.a (mon.0) 512 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:23:55.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:55 smithi100 bash[31715]: cluster 2024-07-28T19:23:54.072995+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e42: 6 total, 6 up, 6 in 2024-07-28T19:23:56.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:56 smithi100 bash[21828]: cluster 2024-07-28T19:23:55.074295+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e43: 6 total, 6 up, 6 in 2024-07-28T19:23:56.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:56 smithi100 bash[31715]: cluster 2024-07-28T19:23:55.074295+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e43: 6 total, 6 up, 6 in 2024-07-28T19:23:56.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:56 smithi138 bash[23952]: cluster 2024-07-28T19:23:55.074295+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e43: 6 total, 6 up, 6 in 2024-07-28T19:23:57.459 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:57 smithi138 bash[23952]: cluster 2024-07-28T19:23:55.388644+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v241: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:57.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:57 smithi100 bash[31715]: cluster 2024-07-28T19:23:55.388644+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v241: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:57.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:57 smithi100 bash[21828]: cluster 2024-07-28T19:23:55.388644+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v241: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:58.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:58 smithi138 bash[23952]: cluster 2024-07-28T19:23:58.091576+0000 mon.a (mon.0) 515 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-07-28T19:23:58.458 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:58 smithi138 bash[23952]: cluster 2024-07-28T19:23:58.091623+0000 mon.a (mon.0) 516 : cluster [INF] Cluster is now healthy 2024-07-28T19:23:58.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:58 smithi100 bash[31715]: cluster 2024-07-28T19:23:58.091576+0000 mon.a (mon.0) 515 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-07-28T19:23:58.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:58 smithi100 bash[31715]: cluster 2024-07-28T19:23:58.091623+0000 mon.a (mon.0) 516 : cluster [INF] Cluster is now healthy 2024-07-28T19:23:58.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:58 smithi100 bash[21828]: cluster 2024-07-28T19:23:58.091576+0000 mon.a (mon.0) 515 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-07-28T19:23:58.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:58 smithi100 bash[21828]: cluster 2024-07-28T19:23:58.091623+0000 mon.a (mon.0) 516 : cluster [INF] Cluster is now healthy 2024-07-28T19:23:59.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:23:59 smithi100 bash[31715]: cluster 2024-07-28T19:23:57.389221+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:59.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:23:59 smithi100 bash[21828]: cluster 2024-07-28T19:23:57.389221+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:59.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:23:59 smithi138 bash[23952]: cluster 2024-07-28T19:23:57.389221+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:23:59.977 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-07-28T19:24:00.149 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-07-28T19:24:00.165 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2024-07-28T19:24:00.165 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2024-07-28T19:24:00.165 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135243 s, 775 MB/s 2024-07-28T19:24:00.165 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2024-07-28T19:24:00.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch daemon add osd smithi138:vg_nvme/lv_2 2024-07-28T19:24:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:01 smithi138 bash[23952]: cluster 2024-07-28T19:23:59.389743+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-07-28T19:24:01.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:01 smithi100 bash[21828]: cluster 2024-07-28T19:23:59.389743+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-07-28T19:24:01.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:01 smithi100 bash[31715]: cluster 2024-07-28T19:23:59.389743+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: cluster 2024-07-28T19:24:01.390323+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: cephadm 2024-07-28T19:24:01.408482+0000 mgr.y (mgr.14150) 265 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: audit 2024-07-28T19:24:01.419114+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: audit 2024-07-28T19:24:01.429463+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: audit 2024-07-28T19:24:01.431530+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: audit 2024-07-28T19:24:01.433428+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: cephadm 2024-07-28T19:24:01.434830+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 8632M 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: audit 2024-07-28T19:24:01.442960+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: audit 2024-07-28T19:24:01.444994+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:02.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: audit 2024-07-28T19:24:01.446769+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:02.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[21828]: audit 2024-07-28T19:24:01.456293+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: cluster 2024-07-28T19:24:01.390323+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-28T19:24:02.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: cephadm 2024-07-28T19:24:01.408482+0000 mgr.y (mgr.14150) 265 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:24:02.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: audit 2024-07-28T19:24:01.419114+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: audit 2024-07-28T19:24:01.429463+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: audit 2024-07-28T19:24:01.431530+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: audit 2024-07-28T19:24:01.433428+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: cephadm 2024-07-28T19:24:01.434830+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 8632M 2024-07-28T19:24:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: audit 2024-07-28T19:24:01.442960+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: audit 2024-07-28T19:24:01.444994+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: audit 2024-07-28T19:24:01.446769+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:02.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:02 smithi100 bash[31715]: audit 2024-07-28T19:24:01.456293+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: cluster 2024-07-28T19:24:01.390323+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: cephadm 2024-07-28T19:24:01.408482+0000 mgr.y (mgr.14150) 265 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: audit 2024-07-28T19:24:01.419114+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: audit 2024-07-28T19:24:01.429463+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: audit 2024-07-28T19:24:01.431530+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: audit 2024-07-28T19:24:01.433428+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: cephadm 2024-07-28T19:24:01.434830+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 8632M 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: audit 2024-07-28T19:24:01.442960+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: audit 2024-07-28T19:24:01.444994+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: audit 2024-07-28T19:24:01.446769+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:02.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:02 smithi138 bash[23952]: audit 2024-07-28T19:24:01.456293+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:04.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:04 smithi100 bash[21828]: cluster 2024-07-28T19:24:03.390866+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-07-28T19:24:04.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:04 smithi100 bash[31715]: cluster 2024-07-28T19:24:03.390866+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-07-28T19:24:05.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:04 smithi138 bash[23952]: cluster 2024-07-28T19:24:03.390866+0000 mgr.y (mgr.14150) 267 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-07-28T19:24:05.652 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:24:06.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:06 smithi100 bash[21828]: cluster 2024-07-28T19:24:05.391432+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-07-28T19:24:06.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:06 smithi100 bash[31715]: cluster 2024-07-28T19:24:05.391432+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-07-28T19:24:07.007 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:06 smithi138 bash[23952]: cluster 2024-07-28T19:24:05.391432+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-07-28T19:24:08.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:08 smithi100 bash[21828]: cluster 2024-07-28T19:24:07.392058+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-28T19:24:08.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:08 smithi100 bash[21828]: audit 2024-07-28T19:24:08.626478+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:24:08.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:08 smithi100 bash[21828]: audit 2024-07-28T19:24:08.632401+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:24:08.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:08 smithi100 bash[21828]: audit 2024-07-28T19:24:08.633894+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:08.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:08 smithi100 bash[31715]: cluster 2024-07-28T19:24:07.392058+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-28T19:24:08.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:08 smithi100 bash[31715]: audit 2024-07-28T19:24:08.626478+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:24:08.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:08 smithi100 bash[31715]: audit 2024-07-28T19:24:08.632401+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:24:08.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:08 smithi100 bash[31715]: audit 2024-07-28T19:24:08.633894+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:09.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:08 smithi138 bash[23952]: cluster 2024-07-28T19:24:07.392058+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-28T19:24:09.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:08 smithi138 bash[23952]: audit 2024-07-28T19:24:08.626478+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:24:09.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:08 smithi138 bash[23952]: audit 2024-07-28T19:24:08.632401+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:24:09.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:08 smithi138 bash[23952]: audit 2024-07-28T19:24:08.633894+0000 mon.a (mon.0) 527 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:10.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:09 smithi138 bash[23952]: audit 2024-07-28T19:24:08.620220+0000 mgr.y (mgr.14150) 270 : audit [DBG] from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:24:10.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:09 smithi100 bash[31715]: audit 2024-07-28T19:24:08.620220+0000 mgr.y (mgr.14150) 270 : audit [DBG] from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:24:10.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:09 smithi100 bash[21828]: audit 2024-07-28T19:24:08.620220+0000 mgr.y (mgr.14150) 270 : audit [DBG] from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:24:11.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:10 smithi138 bash[23952]: cluster 2024-07-28T19:24:09.392609+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:11.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:10 smithi100 bash[21828]: cluster 2024-07-28T19:24:09.392609+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:11.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:10 smithi100 bash[31715]: cluster 2024-07-28T19:24:09.392609+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:13.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:12 smithi138 bash[23952]: cluster 2024-07-28T19:24:11.393178+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:13.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:12 smithi100 bash[21828]: cluster 2024-07-28T19:24:11.393178+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:13.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:12 smithi100 bash[31715]: cluster 2024-07-28T19:24:11.393178+0000 mgr.y (mgr.14150) 272 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:15.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:14 smithi138 bash[23952]: cluster 2024-07-28T19:24:13.393696+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:15.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:14 smithi100 bash[21828]: cluster 2024-07-28T19:24:13.393696+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:15.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:14 smithi100 bash[31715]: cluster 2024-07-28T19:24:13.393696+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:17.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:16 smithi138 bash[23952]: cluster 2024-07-28T19:24:15.394262+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:17.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:16 smithi100 bash[21828]: cluster 2024-07-28T19:24:15.394262+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:17.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:16 smithi100 bash[31715]: cluster 2024-07-28T19:24:15.394262+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:18.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:17 smithi138 bash[23952]: audit 2024-07-28T19:24:16.813848+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.138:0/2600663720' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]: dispatch 2024-07-28T19:24:18.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:17 smithi138 bash[23952]: audit 2024-07-28T19:24:16.815692+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]: dispatch 2024-07-28T19:24:18.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:17 smithi138 bash[23952]: audit 2024-07-28T19:24:16.823334+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]': finished 2024-07-28T19:24:18.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:17 smithi138 bash[23952]: cluster 2024-07-28T19:24:16.823451+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2024-07-28T19:24:18.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:17 smithi138 bash[23952]: audit 2024-07-28T19:24:16.823769+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:18.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:17 smithi138 bash[23952]: audit 2024-07-28T19:24:17.437685+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.138:0/1094073750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:24:18.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[21828]: audit 2024-07-28T19:24:16.813848+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.138:0/2600663720' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]: dispatch 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[21828]: audit 2024-07-28T19:24:16.815692+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]: dispatch 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[21828]: audit 2024-07-28T19:24:16.823334+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]': finished 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[21828]: cluster 2024-07-28T19:24:16.823451+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[21828]: audit 2024-07-28T19:24:16.823769+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[21828]: audit 2024-07-28T19:24:17.437685+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.138:0/1094073750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[31715]: audit 2024-07-28T19:24:16.813848+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.138:0/2600663720' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]: dispatch 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[31715]: audit 2024-07-28T19:24:16.815692+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]: dispatch 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[31715]: audit 2024-07-28T19:24:16.823334+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8a20da48-aee9-44d8-a3db-63f916c6b763"}]': finished 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[31715]: cluster 2024-07-28T19:24:16.823451+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e44: 7 total, 6 up, 7 in 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[31715]: audit 2024-07-28T19:24:16.823769+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:18.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:17 smithi100 bash[31715]: audit 2024-07-28T19:24:17.437685+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.138:0/1094073750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:24:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:18 smithi100 bash[21828]: cluster 2024-07-28T19:24:17.394766+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:19.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:18 smithi100 bash[31715]: cluster 2024-07-28T19:24:17.394766+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:18 smithi138 bash[23952]: cluster 2024-07-28T19:24:17.394766+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:20 smithi100 bash[21828]: cluster 2024-07-28T19:24:19.395235+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:21.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:20 smithi100 bash[31715]: cluster 2024-07-28T19:24:19.395235+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:20 smithi138 bash[23952]: cluster 2024-07-28T19:24:19.395235+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:23.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:22 smithi100 bash[21828]: cluster 2024-07-28T19:24:21.395812+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:23.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:22 smithi100 bash[31715]: cluster 2024-07-28T19:24:21.395812+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:23.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:22 smithi138 bash[23952]: cluster 2024-07-28T19:24:21.395812+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:25.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:24 smithi100 bash[21828]: cluster 2024-07-28T19:24:23.396311+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:25.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:24 smithi100 bash[31715]: cluster 2024-07-28T19:24:23.396311+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:25.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:24 smithi138 bash[23952]: cluster 2024-07-28T19:24:23.396311+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:27.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:26 smithi100 bash[21828]: cluster 2024-07-28T19:24:25.396822+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:27.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:26 smithi100 bash[31715]: cluster 2024-07-28T19:24:25.396822+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:27.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:26 smithi138 bash[23952]: cluster 2024-07-28T19:24:25.396822+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:29.150 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:28 smithi138 bash[23952]: cluster 2024-07-28T19:24:27.397357+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:29.150 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:28 smithi138 bash[23952]: audit 2024-07-28T19:24:28.805180+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-28T19:24:29.150 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:28 smithi138 bash[23952]: audit 2024-07-28T19:24:28.807077+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:29.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:28 smithi100 bash[21828]: cluster 2024-07-28T19:24:27.397357+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:29.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:28 smithi100 bash[21828]: audit 2024-07-28T19:24:28.805180+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-28T19:24:29.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:28 smithi100 bash[21828]: audit 2024-07-28T19:24:28.807077+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:29.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:28 smithi100 bash[31715]: cluster 2024-07-28T19:24:27.397357+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:29.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:28 smithi100 bash[31715]: audit 2024-07-28T19:24:28.805180+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-28T19:24:29.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:28 smithi100 bash[31715]: audit 2024-07-28T19:24:28.807077+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:30.149 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:29 smithi138 bash[23952]: cephadm 2024-07-28T19:24:28.808639+0000 mgr.y (mgr.14150) 281 : cephadm [INF] Deploying daemon osd.6 on smithi138 2024-07-28T19:24:30.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:29 smithi100 bash[21828]: cephadm 2024-07-28T19:24:28.808639+0000 mgr.y (mgr.14150) 281 : cephadm [INF] Deploying daemon osd.6 on smithi138 2024-07-28T19:24:30.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:29 smithi100 bash[31715]: cephadm 2024-07-28T19:24:28.808639+0000 mgr.y (mgr.14150) 281 : cephadm [INF] Deploying daemon osd.6 on smithi138 2024-07-28T19:24:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:30 smithi100 bash[21828]: cluster 2024-07-28T19:24:29.397822+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:31.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:30 smithi100 bash[31715]: cluster 2024-07-28T19:24:29.397822+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:30 smithi138 bash[23952]: cluster 2024-07-28T19:24:29.397822+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:33.007 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:24:32 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:24:33.008 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:32 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:24:33.008 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:32 smithi138 bash[23952]: cluster 2024-07-28T19:24:31.398413+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:33.009 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:24:32 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:24:33.009 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:24:32 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:24:33.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:32 smithi100 bash[21828]: cluster 2024-07-28T19:24:31.398413+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:33.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:32 smithi100 bash[31715]: cluster 2024-07-28T19:24:31.398413+0000 mgr.y (mgr.14150) 283 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:24:33.306 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:24:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:24:33.306 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:24:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:24:33.307 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:24:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:24:34.146 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:33 smithi138 bash[23952]: audit 2024-07-28T19:24:33.150269+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:24:34.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:33 smithi138 bash[23952]: audit 2024-07-28T19:24:33.157751+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:34.147 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:33 smithi138 bash[23952]: audit 2024-07-28T19:24:33.166096+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:33 smithi100 bash[21828]: audit 2024-07-28T19:24:33.150269+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:24:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:33 smithi100 bash[21828]: audit 2024-07-28T19:24:33.157751+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:33 smithi100 bash[21828]: audit 2024-07-28T19:24:33.166096+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:34.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:33 smithi100 bash[31715]: audit 2024-07-28T19:24:33.150269+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:24:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:33 smithi100 bash[31715]: audit 2024-07-28T19:24:33.157751+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:33 smithi100 bash[31715]: audit 2024-07-28T19:24:33.166096+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:35.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:35 smithi100 bash[21828]: cluster 2024-07-28T19:24:33.398909+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:35.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:35 smithi100 bash[31715]: cluster 2024-07-28T19:24:33.398909+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:35.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:35 smithi138 bash[23952]: cluster 2024-07-28T19:24:33.398909+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:37.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:37 smithi100 bash[21828]: cluster 2024-07-28T19:24:35.399467+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:37.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:37 smithi100 bash[31715]: cluster 2024-07-28T19:24:35.399467+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:37.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:37 smithi138 bash[23952]: cluster 2024-07-28T19:24:35.399467+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:39.292 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:39 smithi138 bash[23952]: cluster 2024-07-28T19:24:37.399998+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:39.292 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:39 smithi138 bash[23952]: audit 2024-07-28T19:24:38.255150+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:39.292 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:39 smithi138 bash[23952]: audit 2024-07-28T19:24:38.263651+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:39.293 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:39 smithi138 bash[23952]: audit 2024-07-28T19:24:38.676101+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:39.293 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:39 smithi138 bash[23952]: audit 2024-07-28T19:24:38.677651+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:39.293 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:39 smithi138 bash[23952]: audit 2024-07-28T19:24:38.686105+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:39.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[21828]: cluster 2024-07-28T19:24:37.399998+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:39.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[21828]: audit 2024-07-28T19:24:38.255150+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:39.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[21828]: audit 2024-07-28T19:24:38.263651+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:39.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[21828]: audit 2024-07-28T19:24:38.676101+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:39.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[21828]: audit 2024-07-28T19:24:38.677651+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:39.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[21828]: audit 2024-07-28T19:24:38.686105+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:39.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[31715]: cluster 2024-07-28T19:24:37.399998+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:39.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[31715]: audit 2024-07-28T19:24:38.255150+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:39.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[31715]: audit 2024-07-28T19:24:38.263651+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:39.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[31715]: audit 2024-07-28T19:24:38.676101+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:39.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[31715]: audit 2024-07-28T19:24:38.677651+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:39.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:39 smithi100 bash[31715]: audit 2024-07-28T19:24:38.686105+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:40.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:40 smithi100 bash[21828]: audit 2024-07-28T19:24:39.824898+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-28T19:24:40.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:40 smithi100 bash[21828]: audit 2024-07-28T19:24:39.826563+0000 mon.a (mon.0) 542 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-28T19:24:40.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:40 smithi100 bash[31715]: audit 2024-07-28T19:24:39.824898+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-28T19:24:40.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:40 smithi100 bash[31715]: audit 2024-07-28T19:24:39.826563+0000 mon.a (mon.0) 542 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-28T19:24:40.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:40 smithi138 bash[23952]: audit 2024-07-28T19:24:39.824898+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-28T19:24:40.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:40 smithi138 bash[23952]: audit 2024-07-28T19:24:39.826563+0000 mon.a (mon.0) 542 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-07-28T19:24:41.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:41 smithi138 bash[23952]: cluster 2024-07-28T19:24:39.400596+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:41.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:41 smithi138 bash[23952]: audit 2024-07-28T19:24:40.188257+0000 mon.a (mon.0) 543 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-07-28T19:24:41.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:41 smithi138 bash[23952]: cluster 2024-07-28T19:24:40.188382+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2024-07-28T19:24:41.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:41 smithi138 bash[23952]: audit 2024-07-28T19:24:40.188570+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:24:41.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:41 smithi138 bash[23952]: audit 2024-07-28T19:24:40.188660+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:41.369 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:41 smithi138 bash[23952]: audit 2024-07-28T19:24:40.190364+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:24:41.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[21828]: cluster 2024-07-28T19:24:39.400596+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:41.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[21828]: audit 2024-07-28T19:24:40.188257+0000 mon.a (mon.0) 543 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-07-28T19:24:41.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[21828]: cluster 2024-07-28T19:24:40.188382+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2024-07-28T19:24:41.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[21828]: audit 2024-07-28T19:24:40.188570+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:24:41.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[21828]: audit 2024-07-28T19:24:40.188660+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:41.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[21828]: audit 2024-07-28T19:24:40.190364+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:24:41.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[31715]: cluster 2024-07-28T19:24:39.400596+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:41.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[31715]: audit 2024-07-28T19:24:40.188257+0000 mon.a (mon.0) 543 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-07-28T19:24:41.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[31715]: cluster 2024-07-28T19:24:40.188382+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e45: 7 total, 6 up, 7 in 2024-07-28T19:24:41.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[31715]: audit 2024-07-28T19:24:40.188570+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:24:41.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[31715]: audit 2024-07-28T19:24:40.188660+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:41.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:41 smithi100 bash[31715]: audit 2024-07-28T19:24:40.190364+0000 mon.a (mon.0) 546 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:24:41.721 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 6 on host 'smithi138' 2024-07-28T19:24:42.373 DEBUG:teuthology.orchestra.run.smithi138:osd.6> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.6.service 2024-07-28T19:24:42.376 INFO:tasks.cephadm:Deploying osd.7 on smithi138 with /dev/vg_nvme/lv_1... 2024-07-28T19:24:42.376 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-07-28T19:24:42.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[21828]: audit 2024-07-28T19:24:41.192515+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[21828]: cluster 2024-07-28T19:24:41.192794+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e46: 7 total, 6 up, 7 in 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[21828]: audit 2024-07-28T19:24:41.193167+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[21828]: audit 2024-07-28T19:24:41.198729+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[21828]: cluster 2024-07-28T19:24:41.401163+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[21828]: audit 2024-07-28T19:24:41.700809+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[21828]: audit 2024-07-28T19:24:41.710320+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[21828]: audit 2024-07-28T19:24:41.721196+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[31715]: audit 2024-07-28T19:24:41.192515+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[31715]: cluster 2024-07-28T19:24:41.192794+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e46: 7 total, 6 up, 7 in 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[31715]: audit 2024-07-28T19:24:41.193167+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[31715]: audit 2024-07-28T19:24:41.198729+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:42.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[31715]: cluster 2024-07-28T19:24:41.401163+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:42.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[31715]: audit 2024-07-28T19:24:41.700809+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:24:42.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[31715]: audit 2024-07-28T19:24:41.710320+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:42.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:42 smithi100 bash[31715]: audit 2024-07-28T19:24:41.721196+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:42 smithi138 bash[23952]: audit 2024-07-28T19:24:41.192515+0000 mon.a (mon.0) 547 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:24:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:42 smithi138 bash[23952]: cluster 2024-07-28T19:24:41.192794+0000 mon.a (mon.0) 548 : cluster [DBG] osdmap e46: 7 total, 6 up, 7 in 2024-07-28T19:24:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:42 smithi138 bash[23952]: audit 2024-07-28T19:24:41.193167+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:42 smithi138 bash[23952]: audit 2024-07-28T19:24:41.198729+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:42 smithi138 bash[23952]: cluster 2024-07-28T19:24:41.401163+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-07-28T19:24:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:42 smithi138 bash[23952]: audit 2024-07-28T19:24:41.700809+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:24:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:42 smithi138 bash[23952]: audit 2024-07-28T19:24:41.710320+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:42 smithi138 bash[23952]: audit 2024-07-28T19:24:41.721196+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:43.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[21828]: cluster 2024-07-28T19:24:40.795227+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:24:43.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[21828]: cluster 2024-07-28T19:24:40.795339+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:24:43.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[21828]: cluster 2024-07-28T19:24:42.210691+0000 mon.a (mon.0) 554 : cluster [INF] osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988] boot 2024-07-28T19:24:43.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[21828]: cluster 2024-07-28T19:24:42.210805+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2024-07-28T19:24:43.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[21828]: audit 2024-07-28T19:24:42.213538+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:43.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[21828]: cluster 2024-07-28T19:24:42.747720+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2024-07-28T19:24:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[31715]: cluster 2024-07-28T19:24:40.795227+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:24:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[31715]: cluster 2024-07-28T19:24:40.795339+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:24:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[31715]: cluster 2024-07-28T19:24:42.210691+0000 mon.a (mon.0) 554 : cluster [INF] osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988] boot 2024-07-28T19:24:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[31715]: cluster 2024-07-28T19:24:42.210805+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2024-07-28T19:24:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[31715]: audit 2024-07-28T19:24:42.213538+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:43 smithi100 bash[31715]: cluster 2024-07-28T19:24:42.747720+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2024-07-28T19:24:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:43 smithi138 bash[23952]: cluster 2024-07-28T19:24:40.795227+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:24:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:43 smithi138 bash[23952]: cluster 2024-07-28T19:24:40.795339+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:24:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:43 smithi138 bash[23952]: cluster 2024-07-28T19:24:42.210691+0000 mon.a (mon.0) 554 : cluster [INF] osd.6 [v2:172.21.15.138:6816/3709444988,v1:172.21.15.138:6817/3709444988] boot 2024-07-28T19:24:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:43 smithi138 bash[23952]: cluster 2024-07-28T19:24:42.210805+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e47: 7 total, 7 up, 7 in 2024-07-28T19:24:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:43 smithi138 bash[23952]: audit 2024-07-28T19:24:42.213538+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:24:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:43 smithi138 bash[23952]: cluster 2024-07-28T19:24:42.747720+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e48: 7 total, 7 up, 7 in 2024-07-28T19:24:45.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:44 smithi138 bash[23952]: cluster 2024-07-28T19:24:43.401666+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v270: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:45.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:44 smithi138 bash[23952]: cluster 2024-07-28T19:24:43.746975+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2024-07-28T19:24:45.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:44 smithi100 bash[31715]: cluster 2024-07-28T19:24:43.401666+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v270: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:45.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:44 smithi100 bash[31715]: cluster 2024-07-28T19:24:43.746975+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2024-07-28T19:24:45.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:44 smithi100 bash[21828]: cluster 2024-07-28T19:24:43.401666+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v270: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:45.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:44 smithi100 bash[21828]: cluster 2024-07-28T19:24:43.746975+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e49: 7 total, 7 up, 7 in 2024-07-28T19:24:46.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:45 smithi138 bash[23952]: cluster 2024-07-28T19:24:44.751536+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2024-07-28T19:24:46.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:45 smithi138 bash[23952]: cluster 2024-07-28T19:24:45.751654+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e51: 7 total, 7 up, 7 in 2024-07-28T19:24:46.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:45 smithi100 bash[31715]: cluster 2024-07-28T19:24:44.751536+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2024-07-28T19:24:46.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:45 smithi100 bash[31715]: cluster 2024-07-28T19:24:45.751654+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e51: 7 total, 7 up, 7 in 2024-07-28T19:24:46.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:45 smithi100 bash[21828]: cluster 2024-07-28T19:24:44.751536+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e50: 7 total, 7 up, 7 in 2024-07-28T19:24:46.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:45 smithi100 bash[21828]: cluster 2024-07-28T19:24:45.751654+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e51: 7 total, 7 up, 7 in 2024-07-28T19:24:46.494 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:24:47.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:46 smithi138 bash[23952]: cluster 2024-07-28T19:24:45.402197+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v273: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:47.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:46 smithi100 bash[31715]: cluster 2024-07-28T19:24:45.402197+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v273: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:47.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:46 smithi100 bash[21828]: cluster 2024-07-28T19:24:45.402197+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v273: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:49.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:48 smithi138 bash[23952]: cluster 2024-07-28T19:24:47.402717+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v275: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:49.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:48 smithi100 bash[31715]: cluster 2024-07-28T19:24:47.402717+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v275: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:49.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:48 smithi100 bash[21828]: cluster 2024-07-28T19:24:47.402717+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v275: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:24:51.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:50 smithi138 bash[23952]: cluster 2024-07-28T19:24:49.403281+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-28T19:24:51.228 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-07-28T19:24:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:50 smithi100 bash[31715]: cluster 2024-07-28T19:24:49.403281+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-28T19:24:51.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:50 smithi100 bash[21828]: cluster 2024-07-28T19:24:49.403281+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-28T19:24:51.427 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-07-28T19:24:51.445 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2024-07-28T19:24:51.445 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2024-07-28T19:24:51.446 INFO:teuthology.orchestra.run.smithi138.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0165284 s, 634 MB/s 2024-07-28T19:24:51.446 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2024-07-28T19:24:52.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch daemon add osd smithi138:vg_nvme/lv_1 2024-07-28T19:24:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: cluster 2024-07-28T19:24:51.403908+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-28T19:24:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.649664+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.658291+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.660503+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.662313+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.664009+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.673336+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.675506+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:53.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.677197+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:53.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:52 smithi138 bash[23952]: audit 2024-07-28T19:24:52.687037+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: cluster 2024-07-28T19:24:51.403908+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-28T19:24:53.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.649664+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.658291+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.660503+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.662313+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.664009+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.673336+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.675506+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.677197+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[21828]: audit 2024-07-28T19:24:52.687037+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: cluster 2024-07-28T19:24:51.403908+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.649664+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.658291+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.660503+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.662313+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.664009+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.673336+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.675506+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:24:53.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.677197+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:24:53.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:52 smithi100 bash[31715]: audit 2024-07-28T19:24:52.687037+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:24:54.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:53 smithi138 bash[23952]: cephadm 2024-07-28T19:24:52.641131+0000 mgr.y (mgr.14150) 294 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:24:54.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:53 smithi138 bash[23952]: cephadm 2024-07-28T19:24:52.665321+0000 mgr.y (mgr.14150) 295 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 5755M 2024-07-28T19:24:54.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:53 smithi100 bash[21828]: cephadm 2024-07-28T19:24:52.641131+0000 mgr.y (mgr.14150) 294 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:24:54.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:53 smithi100 bash[21828]: cephadm 2024-07-28T19:24:52.665321+0000 mgr.y (mgr.14150) 295 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 5755M 2024-07-28T19:24:54.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:53 smithi100 bash[31715]: cephadm 2024-07-28T19:24:52.641131+0000 mgr.y (mgr.14150) 294 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:24:54.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:53 smithi100 bash[31715]: cephadm 2024-07-28T19:24:52.665321+0000 mgr.y (mgr.14150) 295 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 5755M 2024-07-28T19:24:55.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:54 smithi138 bash[23952]: cluster 2024-07-28T19:24:53.404577+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-28T19:24:55.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:54 smithi100 bash[21828]: cluster 2024-07-28T19:24:53.404577+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-28T19:24:55.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:54 smithi100 bash[31715]: cluster 2024-07-28T19:24:53.404577+0000 mgr.y (mgr.14150) 296 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-07-28T19:24:57.007 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:24:57.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:56 smithi138 bash[23952]: cluster 2024-07-28T19:24:55.405166+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-28T19:24:57.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:56 smithi100 bash[31715]: cluster 2024-07-28T19:24:55.405166+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-28T19:24:57.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:56 smithi100 bash[21828]: cluster 2024-07-28T19:24:55.405166+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-28T19:24:59.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:24:58 smithi138 bash[23952]: cluster 2024-07-28T19:24:57.405665+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-07-28T19:24:59.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:24:58 smithi100 bash[21828]: cluster 2024-07-28T19:24:57.405665+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-07-28T19:24:59.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:24:58 smithi100 bash[31715]: cluster 2024-07-28T19:24:57.405665+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-07-28T19:25:01.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:00 smithi138 bash[23952]: cluster 2024-07-28T19:24:59.406207+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-28T19:25:01.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:00 smithi138 bash[23952]: audit 2024-07-28T19:25:00.146957+0000 mgr.y (mgr.14150) 300 : audit [DBG] from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:25:01.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:00 smithi138 bash[23952]: audit 2024-07-28T19:25:00.149834+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:25:01.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:00 smithi138 bash[23952]: audit 2024-07-28T19:25:00.155832+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:25:01.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:00 smithi138 bash[23952]: audit 2024-07-28T19:25:00.157441+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:01.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[31715]: cluster 2024-07-28T19:24:59.406207+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[31715]: audit 2024-07-28T19:25:00.146957+0000 mgr.y (mgr.14150) 300 : audit [DBG] from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[31715]: audit 2024-07-28T19:25:00.149834+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[31715]: audit 2024-07-28T19:25:00.155832+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[31715]: audit 2024-07-28T19:25:00.157441+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[21828]: cluster 2024-07-28T19:24:59.406207+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[21828]: audit 2024-07-28T19:25:00.146957+0000 mgr.y (mgr.14150) 300 : audit [DBG] from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[21828]: audit 2024-07-28T19:25:00.149834+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[21828]: audit 2024-07-28T19:25:00.155832+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-07-28T19:25:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:00 smithi100 bash[21828]: audit 2024-07-28T19:25:00.157441+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:03.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:02 smithi100 bash[31715]: cluster 2024-07-28T19:25:01.406748+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:03.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:02 smithi100 bash[21828]: cluster 2024-07-28T19:25:01.406748+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:03.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:02 smithi138 bash[23952]: cluster 2024-07-28T19:25:01.406748+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:05.140 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:04 smithi138 bash[23952]: cluster 2024-07-28T19:25:03.407381+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:05.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:04 smithi100 bash[31715]: cluster 2024-07-28T19:25:03.407381+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:05.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:04 smithi100 bash[21828]: cluster 2024-07-28T19:25:03.407381+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:07.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:06 smithi100 bash[31715]: cluster 2024-07-28T19:25:05.408014+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:07.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:06 smithi100 bash[21828]: cluster 2024-07-28T19:25:05.408014+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:07.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:06 smithi138 bash[23952]: cluster 2024-07-28T19:25:05.408014+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[21828]: cluster 2024-07-28T19:25:07.408575+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[21828]: audit 2024-07-28T19:25:08.433237+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.138:0/513217253' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]: dispatch 2024-07-28T19:25:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[21828]: audit 2024-07-28T19:25:08.435360+0000 mon.a (mon.0) 573 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]: dispatch 2024-07-28T19:25:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[21828]: audit 2024-07-28T19:25:08.444037+0000 mon.a (mon.0) 574 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]': finished 2024-07-28T19:25:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[21828]: cluster 2024-07-28T19:25:08.444156+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2024-07-28T19:25:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[21828]: audit 2024-07-28T19:25:08.444454+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[31715]: cluster 2024-07-28T19:25:07.408575+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:09.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[31715]: audit 2024-07-28T19:25:08.433237+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.138:0/513217253' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]: dispatch 2024-07-28T19:25:09.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[31715]: audit 2024-07-28T19:25:08.435360+0000 mon.a (mon.0) 573 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]: dispatch 2024-07-28T19:25:09.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[31715]: audit 2024-07-28T19:25:08.444037+0000 mon.a (mon.0) 574 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]': finished 2024-07-28T19:25:09.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[31715]: cluster 2024-07-28T19:25:08.444156+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2024-07-28T19:25:09.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:08 smithi100 bash[31715]: audit 2024-07-28T19:25:08.444454+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:09.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:08 smithi138 bash[23952]: cluster 2024-07-28T19:25:07.408575+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:08 smithi138 bash[23952]: audit 2024-07-28T19:25:08.433237+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.138:0/513217253' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]: dispatch 2024-07-28T19:25:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:08 smithi138 bash[23952]: audit 2024-07-28T19:25:08.435360+0000 mon.a (mon.0) 573 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]: dispatch 2024-07-28T19:25:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:08 smithi138 bash[23952]: audit 2024-07-28T19:25:08.444037+0000 mon.a (mon.0) 574 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d52b6b70-462f-42ed-966f-fa14921ba172"}]': finished 2024-07-28T19:25:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:08 smithi138 bash[23952]: cluster 2024-07-28T19:25:08.444156+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e52: 8 total, 7 up, 8 in 2024-07-28T19:25:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:08 smithi138 bash[23952]: audit 2024-07-28T19:25:08.444454+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:10.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:09 smithi100 bash[21828]: audit 2024-07-28T19:25:09.088103+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.138:0/4064371375' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:25:10.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:09 smithi100 bash[31715]: audit 2024-07-28T19:25:09.088103+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.138:0/4064371375' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:25:10.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:09 smithi138 bash[23952]: audit 2024-07-28T19:25:09.088103+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.138:0/4064371375' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-07-28T19:25:11.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:10 smithi100 bash[21828]: cluster 2024-07-28T19:25:09.409132+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:11.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:10 smithi100 bash[31715]: cluster 2024-07-28T19:25:09.409132+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:11.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:10 smithi138 bash[23952]: cluster 2024-07-28T19:25:09.409132+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:13.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:12 smithi100 bash[21828]: cluster 2024-07-28T19:25:11.409658+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:13.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:12 smithi100 bash[31715]: cluster 2024-07-28T19:25:11.409658+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:12 smithi138 bash[23952]: cluster 2024-07-28T19:25:11.409658+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:15.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:14 smithi100 bash[31715]: cluster 2024-07-28T19:25:13.410214+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:15.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:14 smithi100 bash[21828]: cluster 2024-07-28T19:25:13.410214+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:15.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:14 smithi138 bash[23952]: cluster 2024-07-28T19:25:13.410214+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:17.137 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:16 smithi138 bash[23952]: cluster 2024-07-28T19:25:15.410695+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:17.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:16 smithi100 bash[31715]: cluster 2024-07-28T19:25:15.410695+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:17.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:16 smithi100 bash[21828]: cluster 2024-07-28T19:25:15.410695+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:19.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:18 smithi138 bash[23952]: cluster 2024-07-28T19:25:17.411215+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:19.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:18 smithi100 bash[31715]: cluster 2024-07-28T19:25:17.411215+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:19.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:18 smithi100 bash[21828]: cluster 2024-07-28T19:25:17.411215+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:21.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:20 smithi138 bash[23952]: cluster 2024-07-28T19:25:19.411747+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:21.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:20 smithi138 bash[23952]: audit 2024-07-28T19:25:20.533296+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-28T19:25:21.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:20 smithi138 bash[23952]: audit 2024-07-28T19:25:20.534981+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:21.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:20 smithi100 bash[31715]: cluster 2024-07-28T19:25:19.411747+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:21.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:20 smithi100 bash[31715]: audit 2024-07-28T19:25:20.533296+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-28T19:25:21.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:20 smithi100 bash[31715]: audit 2024-07-28T19:25:20.534981+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:20 smithi100 bash[21828]: cluster 2024-07-28T19:25:19.411747+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v292: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:20 smithi100 bash[21828]: audit 2024-07-28T19:25:20.533296+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-28T19:25:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:20 smithi100 bash[21828]: audit 2024-07-28T19:25:20.534981+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:22.002 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:21 smithi138 bash[23952]: cephadm 2024-07-28T19:25:20.536473+0000 mgr.y (mgr.14150) 311 : cephadm [INF] Deploying daemon osd.7 on smithi138 2024-07-28T19:25:22.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:21 smithi100 bash[31715]: cephadm 2024-07-28T19:25:20.536473+0000 mgr.y (mgr.14150) 311 : cephadm [INF] Deploying daemon osd.7 on smithi138 2024-07-28T19:25:22.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:21 smithi100 bash[21828]: cephadm 2024-07-28T19:25:20.536473+0000 mgr.y (mgr.14150) 311 : cephadm [INF] Deploying daemon osd.7 on smithi138 2024-07-28T19:25:23.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:22 smithi100 bash[21828]: cluster 2024-07-28T19:25:21.412370+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:23.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:22 smithi100 bash[31715]: cluster 2024-07-28T19:25:21.412370+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:23.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:22 smithi138 bash[23952]: cluster 2024-07-28T19:25:21.412370+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:24.695 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:24.695 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:24.696 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:24.696 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:24.696 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:25.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:25.056 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:25.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:25.056 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:25.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:25:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:24 smithi138 bash[23952]: cluster 2024-07-28T19:25:23.412943+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:24 smithi138 bash[23952]: audit 2024-07-28T19:25:24.839587+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:25:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:24 smithi138 bash[23952]: audit 2024-07-28T19:25:24.847950+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:24 smithi138 bash[23952]: audit 2024-07-28T19:25:24.859024+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:25.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:24 smithi100 bash[21828]: cluster 2024-07-28T19:25:23.412943+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:25.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:24 smithi100 bash[21828]: audit 2024-07-28T19:25:24.839587+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:25:25.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:24 smithi100 bash[21828]: audit 2024-07-28T19:25:24.847950+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:25.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:24 smithi100 bash[21828]: audit 2024-07-28T19:25:24.859024+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:25.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:24 smithi100 bash[31715]: cluster 2024-07-28T19:25:23.412943+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:25.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:24 smithi100 bash[31715]: audit 2024-07-28T19:25:24.839587+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:25:25.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:24 smithi100 bash[31715]: audit 2024-07-28T19:25:24.847950+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:25.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:24 smithi100 bash[31715]: audit 2024-07-28T19:25:24.859024+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:27.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:26 smithi100 bash[21828]: cluster 2024-07-28T19:25:25.413480+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:27.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:26 smithi100 bash[31715]: cluster 2024-07-28T19:25:25.413480+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:26 smithi138 bash[23952]: cluster 2024-07-28T19:25:25.413480+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v295: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:29.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:28 smithi100 bash[21828]: cluster 2024-07-28T19:25:27.414040+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:29.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:28 smithi100 bash[31715]: cluster 2024-07-28T19:25:27.414040+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:29.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:28 smithi138 bash[23952]: cluster 2024-07-28T19:25:27.414040+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:31.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[31715]: cluster 2024-07-28T19:25:29.414632+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:31.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[31715]: audit 2024-07-28T19:25:29.961088+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:31.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[31715]: audit 2024-07-28T19:25:29.986253+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:31.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[31715]: audit 2024-07-28T19:25:29.988084+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:31.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[31715]: audit 2024-07-28T19:25:29.989899+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:25:31.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[31715]: audit 2024-07-28T19:25:29.999914+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[21828]: cluster 2024-07-28T19:25:29.414632+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[21828]: audit 2024-07-28T19:25:29.961088+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[21828]: audit 2024-07-28T19:25:29.986253+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[21828]: audit 2024-07-28T19:25:29.988084+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[21828]: audit 2024-07-28T19:25:29.989899+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:25:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:30 smithi100 bash[21828]: audit 2024-07-28T19:25:29.999914+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:30 smithi138 bash[23952]: cluster 2024-07-28T19:25:29.414632+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:30 smithi138 bash[23952]: audit 2024-07-28T19:25:29.961088+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:30 smithi138 bash[23952]: audit 2024-07-28T19:25:29.986253+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:30 smithi138 bash[23952]: audit 2024-07-28T19:25:29.988084+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:30 smithi138 bash[23952]: audit 2024-07-28T19:25:29.989899+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:25:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:30 smithi138 bash[23952]: audit 2024-07-28T19:25:29.999914+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:32.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:31 smithi100 bash[21828]: audit 2024-07-28T19:25:31.372448+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-28T19:25:32.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:31 smithi100 bash[21828]: audit 2024-07-28T19:25:31.374742+0000 mon.a (mon.0) 587 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-28T19:25:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:31 smithi100 bash[31715]: audit 2024-07-28T19:25:31.372448+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-28T19:25:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:31 smithi100 bash[31715]: audit 2024-07-28T19:25:31.374742+0000 mon.a (mon.0) 587 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-28T19:25:32.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:31 smithi138 bash[23952]: audit 2024-07-28T19:25:31.372448+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-28T19:25:32.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:31 smithi138 bash[23952]: audit 2024-07-28T19:25:31.374742+0000 mon.a (mon.0) 587 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-07-28T19:25:33.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[21828]: cluster 2024-07-28T19:25:31.415275+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:33.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[21828]: audit 2024-07-28T19:25:31.973382+0000 mon.a (mon.0) 588 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-07-28T19:25:33.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[21828]: audit 2024-07-28T19:25:31.973545+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:25:33.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[21828]: cluster 2024-07-28T19:25:31.973577+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2024-07-28T19:25:33.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[21828]: audit 2024-07-28T19:25:31.973826+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:33.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[21828]: audit 2024-07-28T19:25:31.976042+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:25:33.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[31715]: cluster 2024-07-28T19:25:31.415275+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:33.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[31715]: audit 2024-07-28T19:25:31.973382+0000 mon.a (mon.0) 588 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-07-28T19:25:33.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[31715]: audit 2024-07-28T19:25:31.973545+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:25:33.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[31715]: cluster 2024-07-28T19:25:31.973577+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2024-07-28T19:25:33.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[31715]: audit 2024-07-28T19:25:31.973826+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:33.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:32 smithi100 bash[31715]: audit 2024-07-28T19:25:31.976042+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:25:33.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:32 smithi138 bash[23952]: cluster 2024-07-28T19:25:31.415275+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:32 smithi138 bash[23952]: audit 2024-07-28T19:25:31.973382+0000 mon.a (mon.0) 588 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-07-28T19:25:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:32 smithi138 bash[23952]: audit 2024-07-28T19:25:31.973545+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:25:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:32 smithi138 bash[23952]: cluster 2024-07-28T19:25:31.973577+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e53: 8 total, 7 up, 8 in 2024-07-28T19:25:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:32 smithi138 bash[23952]: audit 2024-07-28T19:25:31.973826+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:32 smithi138 bash[23952]: audit 2024-07-28T19:25:31.976042+0000 mon.a (mon.0) 591 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:25:33.333 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 7 on host 'smithi138' 2024-07-28T19:25:33.939 DEBUG:teuthology.orchestra.run.smithi138:osd.7> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.7.service 2024-07-28T19:25:33.942 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-07-28T19:25:33.942 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd stat -f json 2024-07-28T19:25:34.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: audit 2024-07-28T19:25:32.977251+0000 mon.a (mon.0) 592 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:25:34.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: cluster 2024-07-28T19:25:32.977493+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e54: 8 total, 7 up, 8 in 2024-07-28T19:25:34.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: audit 2024-07-28T19:25:32.978169+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: audit 2024-07-28T19:25:32.988948+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: audit 2024-07-28T19:25:33.312656+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: audit 2024-07-28T19:25:33.322620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: audit 2024-07-28T19:25:33.333590+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: cluster 2024-07-28T19:25:33.977675+0000 mon.a (mon.0) 599 : cluster [INF] osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219] boot 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: cluster 2024-07-28T19:25:33.977729+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[21828]: audit 2024-07-28T19:25:33.978051+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: audit 2024-07-28T19:25:32.977251+0000 mon.a (mon.0) 592 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: cluster 2024-07-28T19:25:32.977493+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e54: 8 total, 7 up, 8 in 2024-07-28T19:25:34.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: audit 2024-07-28T19:25:32.978169+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: audit 2024-07-28T19:25:32.988948+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: audit 2024-07-28T19:25:33.312656+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:25:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: audit 2024-07-28T19:25:33.322620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: audit 2024-07-28T19:25:33.333590+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: cluster 2024-07-28T19:25:33.977675+0000 mon.a (mon.0) 599 : cluster [INF] osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219] boot 2024-07-28T19:25:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: cluster 2024-07-28T19:25:33.977729+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-07-28T19:25:34.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:33 smithi100 bash[31715]: audit 2024-07-28T19:25:33.978051+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: audit 2024-07-28T19:25:32.977251+0000 mon.a (mon.0) 592 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: cluster 2024-07-28T19:25:32.977493+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e54: 8 total, 7 up, 8 in 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: audit 2024-07-28T19:25:32.978169+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: audit 2024-07-28T19:25:32.988948+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: audit 2024-07-28T19:25:33.312656+0000 mon.a (mon.0) 596 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: audit 2024-07-28T19:25:33.322620+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: audit 2024-07-28T19:25:33.333590+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: cluster 2024-07-28T19:25:33.977675+0000 mon.a (mon.0) 599 : cluster [INF] osd.7 [v2:172.21.15.138:6824/1158882219,v1:172.21.15.138:6825/1158882219] boot 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: cluster 2024-07-28T19:25:33.977729+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-07-28T19:25:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:33 smithi138 bash[23952]: audit 2024-07-28T19:25:33.978051+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:25:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:34 smithi100 bash[21828]: cluster 2024-07-28T19:25:32.389005+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:25:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:34 smithi100 bash[21828]: cluster 2024-07-28T19:25:32.389076+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:25:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:34 smithi100 bash[21828]: cluster 2024-07-28T19:25:33.415844+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:34 smithi100 bash[21828]: cluster 2024-07-28T19:25:34.980716+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-07-28T19:25:35.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:34 smithi100 bash[31715]: cluster 2024-07-28T19:25:32.389005+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:25:35.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:34 smithi100 bash[31715]: cluster 2024-07-28T19:25:32.389076+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:25:35.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:34 smithi100 bash[31715]: cluster 2024-07-28T19:25:33.415844+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:35.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:34 smithi100 bash[31715]: cluster 2024-07-28T19:25:34.980716+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-07-28T19:25:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:34 smithi138 bash[23952]: cluster 2024-07-28T19:25:32.389005+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-07-28T19:25:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:34 smithi138 bash[23952]: cluster 2024-07-28T19:25:32.389076+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-07-28T19:25:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:34 smithi138 bash[23952]: cluster 2024-07-28T19:25:33.415844+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-07-28T19:25:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:34 smithi138 bash[23952]: cluster 2024-07-28T19:25:34.980716+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-07-28T19:25:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:37 smithi138 bash[23952]: cluster 2024-07-28T19:25:35.416434+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:37 smithi138 bash[23952]: cluster 2024-07-28T19:25:35.996418+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-07-28T19:25:37.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:37 smithi100 bash[21828]: cluster 2024-07-28T19:25:35.416434+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:37.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:37 smithi100 bash[21828]: cluster 2024-07-28T19:25:35.996418+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-07-28T19:25:37.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:37 smithi100 bash[31715]: cluster 2024-07-28T19:25:35.416434+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:37.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:37 smithi100 bash[31715]: cluster 2024-07-28T19:25:35.996418+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-07-28T19:25:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:38 smithi138 bash[23952]: cluster 2024-07-28T19:25:36.999444+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-07-28T19:25:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:38 smithi100 bash[21828]: cluster 2024-07-28T19:25:36.999444+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-07-28T19:25:38.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:38 smithi100 bash[31715]: cluster 2024-07-28T19:25:36.999444+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-07-28T19:25:38.706 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:25:39.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:39 smithi100 bash[21828]: cluster 2024-07-28T19:25:37.416935+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:39.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:39 smithi100 bash[21828]: cluster 2024-07-28T19:25:38.022837+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-07-28T19:25:39.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:39 smithi100 bash[31715]: cluster 2024-07-28T19:25:37.416935+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:39.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:39 smithi100 bash[31715]: cluster 2024-07-28T19:25:38.022837+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-07-28T19:25:39.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:39 smithi138 bash[23952]: cluster 2024-07-28T19:25:37.416935+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:39.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:39 smithi138 bash[23952]: cluster 2024-07-28T19:25:38.022837+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-07-28T19:25:41.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:41 smithi100 bash[21828]: cluster 2024-07-28T19:25:39.417517+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s, 0 objects/s recovering 2024-07-28T19:25:41.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:41 smithi100 bash[31715]: cluster 2024-07-28T19:25:39.417517+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s, 0 objects/s recovering 2024-07-28T19:25:41.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:41 smithi138 bash[23952]: cluster 2024-07-28T19:25:39.417517+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 130 KiB/s, 0 objects/s recovering 2024-07-28T19:25:42.193 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:25:42.896 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1722194733,"num_in_osds":8,"osd_in_since":1722194708,"num_remapped_pgs":0} 2024-07-28T19:25:42.897 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd dump --format=json 2024-07-28T19:25:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: cluster 2024-07-28T19:25:41.418087+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-28T19:25:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.197981+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.100:0/327194337' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-07-28T19:25:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.479047+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.489994+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.492645+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.494525+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.496173+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.497788+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.509584+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.511542+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.513123+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[31715]: audit 2024-07-28T19:25:42.522842+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: cluster 2024-07-28T19:25:41.418087+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-28T19:25:43.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.197981+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.100:0/327194337' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.479047+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.489994+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.492645+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.494525+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.496173+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.497788+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.509584+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.511542+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.513123+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:25:43.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:43 smithi100 bash[21828]: audit 2024-07-28T19:25:42.522842+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: cluster 2024-07-28T19:25:41.418087+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.197981+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.100:0/327194337' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.479047+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.489994+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.492645+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.494525+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.496173+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.497788+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.509584+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:43.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.511542+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:25:43.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.513123+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:25:43.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:43 smithi138 bash[23952]: audit 2024-07-28T19:25:42.522842+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:25:44.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:44 smithi138 bash[23952]: cephadm 2024-07-28T19:25:42.468480+0000 mgr.y (mgr.14150) 323 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:25:44.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:44 smithi138 bash[23952]: cephadm 2024-07-28T19:25:42.499056+0000 mgr.y (mgr.14150) 324 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 4316M 2024-07-28T19:25:44.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:44 smithi100 bash[21828]: cephadm 2024-07-28T19:25:42.468480+0000 mgr.y (mgr.14150) 323 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:25:44.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:44 smithi100 bash[21828]: cephadm 2024-07-28T19:25:42.499056+0000 mgr.y (mgr.14150) 324 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 4316M 2024-07-28T19:25:44.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:44 smithi100 bash[31715]: cephadm 2024-07-28T19:25:42.468480+0000 mgr.y (mgr.14150) 323 : cephadm [INF] Detected new or changed devices on smithi138 2024-07-28T19:25:44.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:44 smithi100 bash[31715]: cephadm 2024-07-28T19:25:42.499056+0000 mgr.y (mgr.14150) 324 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 4316M 2024-07-28T19:25:45.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:45 smithi138 bash[23952]: cluster 2024-07-28T19:25:43.418662+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-07-28T19:25:45.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:45 smithi100 bash[21828]: cluster 2024-07-28T19:25:43.418662+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-07-28T19:25:45.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:45 smithi100 bash[31715]: cluster 2024-07-28T19:25:43.418662+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-07-28T19:25:47.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:47 smithi138 bash[23952]: cluster 2024-07-28T19:25:45.419266+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-07-28T19:25:47.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:47 smithi100 bash[31715]: cluster 2024-07-28T19:25:45.419266+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-07-28T19:25:47.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:47 smithi100 bash[21828]: cluster 2024-07-28T19:25:45.419266+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-07-28T19:25:47.660 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:25:49.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:49 smithi138 bash[23952]: cluster 2024-07-28T19:25:47.419831+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-28T19:25:49.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:49 smithi100 bash[21828]: cluster 2024-07-28T19:25:47.419831+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-28T19:25:49.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:49 smithi100 bash[31715]: cluster 2024-07-28T19:25:47.419831+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-07-28T19:25:50.823 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:25:50.824 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":59,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","created":"2024-07-28T19:16:00.842363+0000","modified":"2024-07-28T19:25:38.003201+0000","last_up_change":"2024-07-28T19:25:33.970932+0000","last_in_change":"2024-07-28T19:25:08.436673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-28T19:21:19.886730+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"c4a49140-8022-42e2-bba9-333eb0e88545","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6803","nonce":2528423215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6805","nonce":2528423215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6809","nonce":2528423215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6807","nonce":2528423215}]},"public_addr":"172.21.15.100:6803/2528423215","cluster_addr":"172.21.15.100:6805/2528423215","heartbeat_back_addr":"172.21.15.100:6809/2528423215","heartbeat_front_addr":"172.21.15.100:6807/2528423215","state":["exists","up"]},{"osd":1,"uuid":"4117fbb5-c88e-4d12-bc76-77050b2c0425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6811","nonce":3874116596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6813","nonce":3874116596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6817","nonce":3874116596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6815","nonce":3874116596}]},"public_addr":"172.21.15.100:6811/3874116596","cluster_addr":"172.21.15.100:6813/3874116596","heartbeat_back_addr":"172.21.15.100:6817/3874116596","heartbeat_front_addr":"172.21.15.100:6815/3874116596","state":["exists","up"]},{"osd":2,"uuid":"f3416396-83c7-42fb-9153-9cb806281347","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6819","nonce":2139838255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6821","nonce":2139838255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6825","nonce":2139838255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6823","nonce":2139838255}]},"public_addr":"172.21.15.100:6819/2139838255","cluster_addr":"172.21.15.100:6821/2139838255","heartbeat_back_addr":"172.21.15.100:6825/2139838255","heartbeat_front_addr":"172.21.15.100:6823/2139838255","state":["exists","up"]},{"osd":3,"uuid":"0363a205-8262-42a4-aaaf-e52558fec7f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6827","nonce":684533287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6829","nonce":684533287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6832","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6833","nonce":684533287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6831","nonce":684533287}]},"public_addr":"172.21.15.100:6827/684533287","cluster_addr":"172.21.15.100:6829/684533287","heartbeat_back_addr":"172.21.15.100:6833/684533287","heartbeat_front_addr":"172.21.15.100:6831/684533287","state":["exists","up"]},{"osd":4,"uuid":"75f51136-e1cc-4d3a-89d8-1618aaff366e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6801","nonce":1306402242}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6803","nonce":1306402242}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6807","nonce":1306402242}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6805","nonce":1306402242}]},"public_addr":"172.21.15.138:6801/1306402242","cluster_addr":"172.21.15.138:6803/1306402242","heartbeat_back_addr":"172.21.15.138:6807/1306402242","heartbeat_front_addr":"172.21.15.138:6805/1306402242","state":["exists","up"]},{"osd":5,"uuid":"0a3e4950-a201-4105-9e45-ef45ce2d5f3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6809","nonce":1102784487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6811","nonce":1102784487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6815","nonce":1102784487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6813","nonce":1102784487}]},"public_addr":"172.21.15.138:6809/1102784487","cluster_addr":"172.21.15.138:6811/1102784487","heartbeat_back_addr":"172.21.15.138:6815/1102784487","heartbeat_front_addr":"172.21.15.138:6813/1102784487","state":["exists","up"]},{"osd":6,"uuid":"8a20da48-aee9-44d8-a3db-63f916c6b763","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6817","nonce":3709444988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6819","nonce":3709444988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6823","nonce":3709444988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6821","nonce":3709444988}]},"public_addr":"172.21.15.138:6817/3709444988","cluster_addr":"172.21.15.138:6819/3709444988","heartbeat_back_addr":"172.21.15.138:6823/3709444988","heartbeat_front_addr":"172.21.15.138:6821/3709444988","state":["exists","up"]},{"osd":7,"uuid":"d52b6b70-462f-42ed-966f-fa14921ba172","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6825","nonce":1158882219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6827","nonce":1158882219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6831","nonce":1158882219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6829","nonce":1158882219}]},"public_addr":"172.21.15.138:6825/1158882219","cluster_addr":"172.21.15.138:6827/1158882219","heartbeat_back_addr":"172.21.15.138:6831/1158882219","heartbeat_front_addr":"172.21.15.138:6829/1158882219","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:19:30.916069+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:20:23.352940+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:21:16.853645+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:09.540873+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:59.198788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:23:49.135324+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:24:40.795346+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:25:32.389079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.100:0/3152621533":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6801/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/553075364":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3841450082":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/989116380":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2863303498":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/2291421462":"2024-07-29T19:16:28.286850+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-28T19:25:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:51 smithi100 bash[31715]: cluster 2024-07-28T19:25:49.420409+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-07-28T19:25:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:51 smithi100 bash[31715]: audit 2024-07-28T19:25:50.826974+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.100:0/3932400107' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:25:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:51 smithi100 bash[21828]: cluster 2024-07-28T19:25:49.420409+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-07-28T19:25:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:51 smithi100 bash[21828]: audit 2024-07-28T19:25:50.826974+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.100:0/3932400107' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:25:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:51 smithi138 bash[23952]: cluster 2024-07-28T19:25:49.420409+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s, 0 objects/s recovering 2024-07-28T19:25:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:51 smithi138 bash[23952]: audit 2024-07-28T19:25:50.826974+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.100:0/3932400107' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:25:51.461 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-07-28T19:21:19.886730+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-07-28T19:25:51.462 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-07-28T19:25:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:53 smithi138 bash[23952]: cluster 2024-07-28T19:25:51.421114+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:53 smithi100 bash[21828]: cluster 2024-07-28T19:25:51.421114+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:53.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:53 smithi100 bash[31715]: cluster 2024-07-28T19:25:51.421114+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:55.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:55 smithi138 bash[23952]: cluster 2024-07-28T19:25:53.421690+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:55.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:55 smithi100 bash[21828]: cluster 2024-07-28T19:25:53.421690+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:55.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:55 smithi100 bash[31715]: cluster 2024-07-28T19:25:53.421690+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:56.222 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:25:57.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:57 smithi138 bash[23952]: cluster 2024-07-28T19:25:55.422244+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:57.430 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:57 smithi100 bash[21828]: cluster 2024-07-28T19:25:55.422244+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:57.430 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:57 smithi100 bash[31715]: cluster 2024-07-28T19:25:55.422244+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:59.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:25:59 smithi138 bash[23952]: cluster 2024-07-28T19:25:57.422788+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:59.470 INFO:teuthology.orchestra.run.smithi100.stdout:pg_num: 1 2024-07-28T19:25:59.480 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:25:59 smithi100 bash[31715]: cluster 2024-07-28T19:25:57.422788+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:25:59.480 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:25:59 smithi100 bash[21828]: cluster 2024-07-28T19:25:57.422788+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:00.129 INFO:tasks.cephadm:Adding prometheus.a on smithi138 2024-07-28T19:26:00.129 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch apply prometheus '1;smithi138=a' 2024-07-28T19:26:00.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:00 smithi100 bash[21828]: audit 2024-07-28T19:25:59.473218+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.100:0/1777100744' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-07-28T19:26:00.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:00 smithi100 bash[31715]: audit 2024-07-28T19:25:59.473218+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.100:0/1777100744' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-07-28T19:26:00.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:00 smithi138 bash[23952]: audit 2024-07-28T19:25:59.473218+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.100:0/1777100744' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-07-28T19:26:01.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:01 smithi100 bash[21828]: cluster 2024-07-28T19:25:59.423312+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:01.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:01 smithi100 bash[31715]: cluster 2024-07-28T19:25:59.423312+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:01.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:01 smithi138 bash[23952]: cluster 2024-07-28T19:25:59.423312+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:03.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:03 smithi100 bash[21828]: cluster 2024-07-28T19:26:01.423857+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:03.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:03 smithi100 bash[31715]: cluster 2024-07-28T19:26:01.423857+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:03.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:03 smithi138 bash[23952]: cluster 2024-07-28T19:26:01.423857+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:04.878 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:26:05.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:05 smithi138 bash[23952]: cluster 2024-07-28T19:26:03.424390+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:05.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:05 smithi100 bash[21828]: cluster 2024-07-28T19:26:03.424390+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:05.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:05 smithi100 bash[31715]: cluster 2024-07-28T19:26:03.424390+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:07.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:07 smithi100 bash[21828]: cluster 2024-07-28T19:26:05.424968+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:07.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:07 smithi100 bash[31715]: cluster 2024-07-28T19:26:05.424968+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:07.532 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:07 smithi138 bash[23952]: cluster 2024-07-28T19:26:05.424968+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:07.883 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled prometheus update... 2024-07-28T19:26:08.534 DEBUG:teuthology.orchestra.run.smithi138:prometheus.a> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@prometheus.a.service 2024-07-28T19:26:08.536 INFO:tasks.cephadm:Adding node-exporter.a on smithi100 2024-07-28T19:26:08.536 INFO:tasks.cephadm:Adding node-exporter.b on smithi138 2024-07-28T19:26:08.536 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch apply node-exporter '2;smithi100=a;smithi138=b' 2024-07-28T19:26:08.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:08 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[21828]: cluster 2024-07-28T19:26:07.425523+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[21828]: audit 2024-07-28T19:26:07.876353+0000 mgr.y (mgr.14150) 338 : audit [DBG] from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[21828]: cephadm 2024-07-28T19:26:07.879265+0000 mgr.y (mgr.14150) 339 : cephadm [INF] Saving service prometheus spec with placement smithi138=a;count:1 2024-07-28T19:26:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[21828]: audit 2024-07-28T19:26:07.887972+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[21828]: audit 2024-07-28T19:26:07.889829+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[21828]: audit 2024-07-28T19:26:08.373832+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[21828]: audit 2024-07-28T19:26:08.375375+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[21828]: audit 2024-07-28T19:26:08.385191+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[31715]: cluster 2024-07-28T19:26:07.425523+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[31715]: audit 2024-07-28T19:26:07.876353+0000 mgr.y (mgr.14150) 338 : audit [DBG] from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[31715]: cephadm 2024-07-28T19:26:07.879265+0000 mgr.y (mgr.14150) 339 : cephadm [INF] Saving service prometheus spec with placement smithi138=a;count:1 2024-07-28T19:26:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[31715]: audit 2024-07-28T19:26:07.887972+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[31715]: audit 2024-07-28T19:26:07.889829+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[31715]: audit 2024-07-28T19:26:08.373832+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[31715]: audit 2024-07-28T19:26:08.375375+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:08 smithi100 bash[31715]: audit 2024-07-28T19:26:08.385191+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:09.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:08 smithi138 bash[23952]: cluster 2024-07-28T19:26:07.425523+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:08 smithi138 bash[23952]: audit 2024-07-28T19:26:07.876353+0000 mgr.y (mgr.14150) 338 : audit [DBG] from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:08 smithi138 bash[23952]: cephadm 2024-07-28T19:26:07.879265+0000 mgr.y (mgr.14150) 339 : cephadm [INF] Saving service prometheus spec with placement smithi138=a;count:1 2024-07-28T19:26:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:08 smithi138 bash[23952]: audit 2024-07-28T19:26:07.887972+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:08 smithi138 bash[23952]: audit 2024-07-28T19:26:07.889829+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:08 smithi138 bash[23952]: audit 2024-07-28T19:26:08.373832+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:08 smithi138 bash[23952]: audit 2024-07-28T19:26:08.375375+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:08 smithi138 bash[23952]: audit 2024-07-28T19:26:08.385191+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:10.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:09 smithi100 bash[21828]: cephadm 2024-07-28T19:26:08.417177+0000 mgr.y (mgr.14150) 340 : cephadm [INF] Deploying daemon prometheus.a on smithi138 2024-07-28T19:26:10.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:09 smithi100 bash[31715]: cephadm 2024-07-28T19:26:08.417177+0000 mgr.y (mgr.14150) 340 : cephadm [INF] Deploying daemon prometheus.a on smithi138 2024-07-28T19:26:10.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:09 smithi138 bash[23952]: cephadm 2024-07-28T19:26:08.417177+0000 mgr.y (mgr.14150) 340 : cephadm [INF] Deploying daemon prometheus.a on smithi138 2024-07-28T19:26:11.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:10 smithi100 bash[21828]: cluster 2024-07-28T19:26:09.426078+0000 mgr.y (mgr.14150) 341 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:11.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:10 smithi100 bash[31715]: cluster 2024-07-28T19:26:09.426078+0000 mgr.y (mgr.14150) 341 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:11.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:10 smithi138 bash[23952]: cluster 2024-07-28T19:26:09.426078+0000 mgr.y (mgr.14150) 341 : cluster [DBG] pgmap v324: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:12 smithi100 bash[21828]: cluster 2024-07-28T19:26:11.426717+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:12 smithi100 bash[31715]: cluster 2024-07-28T19:26:11.426717+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:12 smithi138 bash[23952]: cluster 2024-07-28T19:26:11.426717+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v325: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:13.334 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:26:15.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:14 smithi100 bash[21828]: cluster 2024-07-28T19:26:13.427379+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:14 smithi100 bash[31715]: cluster 2024-07-28T19:26:13.427379+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:15.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:14 smithi138 bash[23952]: cluster 2024-07-28T19:26:13.427379+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v326: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:17.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:16 smithi138 bash[23952]: cluster 2024-07-28T19:26:15.428061+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:17.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:16 smithi100 bash[21828]: cluster 2024-07-28T19:26:15.428061+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:17.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:16 smithi100 bash[31715]: cluster 2024-07-28T19:26:15.428061+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v327: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:19.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:18 smithi100 bash[21828]: cluster 2024-07-28T19:26:17.428711+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:19.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:18 smithi100 bash[31715]: cluster 2024-07-28T19:26:17.428711+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:19.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:18 smithi138 bash[23952]: cluster 2024-07-28T19:26:17.428711+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v328: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:19.704 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled node-exporter update... 2024-07-28T19:26:20.933 DEBUG:teuthology.orchestra.run.smithi100:node-exporter.a> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.a.service 2024-07-28T19:26:20.936 DEBUG:teuthology.orchestra.run.smithi138:node-exporter.b> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.b.service 2024-07-28T19:26:20.939 INFO:tasks.cephadm:Adding alertmanager.a on smithi100 2024-07-28T19:26:20.939 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch apply alertmanager '1;smithi100=a' 2024-07-28T19:26:20.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:20 smithi100 bash[21828]: cluster 2024-07-28T19:26:19.429335+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:20.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:20 smithi100 bash[21828]: audit 2024-07-28T19:26:19.699081+0000 mgr.y (mgr.14150) 347 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi100=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:20.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:20 smithi100 bash[21828]: cephadm 2024-07-28T19:26:19.702109+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Saving service node-exporter spec with placement smithi100=a;smithi138=b;count:2 2024-07-28T19:26:20.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:20 smithi100 bash[21828]: audit 2024-07-28T19:26:19.709455+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:20.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:20 smithi100 bash[31715]: cluster 2024-07-28T19:26:19.429335+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:20.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:20 smithi100 bash[31715]: audit 2024-07-28T19:26:19.699081+0000 mgr.y (mgr.14150) 347 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi100=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:20.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:20 smithi100 bash[31715]: cephadm 2024-07-28T19:26:19.702109+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Saving service node-exporter spec with placement smithi100=a;smithi138=b;count:2 2024-07-28T19:26:20.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:20 smithi100 bash[31715]: audit 2024-07-28T19:26:19.709455+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:20.996 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:20 smithi138 bash[23952]: cluster 2024-07-28T19:26:19.429335+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v329: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:20.996 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:20 smithi138 bash[23952]: audit 2024-07-28T19:26:19.699081+0000 mgr.y (mgr.14150) 347 : audit [DBG] from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi100=a;smithi138=b", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:20.996 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:20 smithi138 bash[23952]: cephadm 2024-07-28T19:26:19.702109+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Saving service node-exporter spec with placement smithi100=a;smithi138=b;count:2 2024-07-28T19:26:20.996 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:20 smithi138 bash[23952]: audit 2024-07-28T19:26:19.709455+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:22 smithi100 bash[21828]: cluster 2024-07-28T19:26:21.429934+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:22.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:22 smithi100 bash[31715]: cluster 2024-07-28T19:26:21.429934+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:23.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:22 smithi138 bash[23952]: cluster 2024-07-28T19:26:21.429934+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v330: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:24.265 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.266 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.266 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.266 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.267 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.267 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.267 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.556 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.556 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:24.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:24 smithi138 systemd[1]: Started Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:26:24.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[21828]: cluster 2024-07-28T19:26:23.430485+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:24.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[21828]: audit 2024-07-28T19:26:24.543092+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:24.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[21828]: audit 2024-07-28T19:26:24.552272+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:24.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[21828]: audit 2024-07-28T19:26:24.560342+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:24.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[21828]: audit 2024-07-28T19:26:24.567184+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-07-28T19:26:24.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[31715]: cluster 2024-07-28T19:26:23.430485+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:24.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[31715]: audit 2024-07-28T19:26:24.543092+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:24.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[31715]: audit 2024-07-28T19:26:24.552272+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:24.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[31715]: audit 2024-07-28T19:26:24.560342+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:24.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:24 smithi100 bash[31715]: audit 2024-07-28T19:26:24.567184+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-07-28T19:26:25.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:24 smithi138 bash[23952]: cluster 2024-07-28T19:26:23.430485+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v331: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:25.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:24 smithi138 bash[23952]: audit 2024-07-28T19:26:24.543092+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:25.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:24 smithi138 bash[23952]: audit 2024-07-28T19:26:24.552272+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:25.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:24 smithi138 bash[23952]: audit 2024-07-28T19:26:24.560342+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' 2024-07-28T19:26:25.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:24 smithi138 bash[23952]: audit 2024-07-28T19:26:24.567184+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-07-28T19:26:25.735 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:26:25.870 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[24937]: ignoring --setuser ceph since I am not root 2024-07-28T19:26:25.870 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[24937]: ignoring --setgroup ceph since I am not root 2024-07-28T19:26:25.870 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[24937]: debug 2024-07-28T19:26:25.615+0000 7f8f0a363640 1 -- 172.21.15.138:0/3645540419 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5567cac729c0 con 0x5567cac7ec00 2024-07-28T19:26:25.871 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[24937]: debug 2024-07-28T19:26:25.723+0000 7f8f0cbbdec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.789Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.789Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.789Z caller=main.go:570 level=info host_details="(Linux 5.15.0-117-generic #127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024 x86_64 smithi138 (none))" 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.789Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.789Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.790Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.791Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.792Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-07-28T19:26:25.871 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.792Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.796Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.796Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.661µs 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.796Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.796Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.796Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.902µs wal_replay_duration=218.822µs wbl_replay_duration=311ns total_replay_duration=260.427µs 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.797Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.797Z caller=main.go:1029 level=info msg="TSDB started" 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.797Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.822Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.406482ms db_storage=1.088µs remote_storage=1.411µs web_handler=476ns query_engine=843ns scrape=379.004µs scrape_sd=24.8µs notify=947ns notify_sd=2.449µs rules=23.730416ms tracing=6.971µs 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.822Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-07-28T19:26:25.872 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:25 smithi138 bash[48170]: ts=2024-07-28T19:26:25.822Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-07-28T19:26:25.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:25 smithi100 bash[22098]: ignoring --setuser ceph since I am not root 2024-07-28T19:26:25.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:25 smithi100 bash[22098]: ignoring --setgroup ceph since I am not root 2024-07-28T19:26:25.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:25 smithi100 bash[22098]: debug 2024-07-28T19:26:25.615+0000 7fb356943640 1 -- 172.21.15.100:0/2158626960 <== mon.0 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5586925109c0 con 0x55869251cc00 2024-07-28T19:26:25.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:25 smithi100 bash[22098]: debug 2024-07-28T19:26:25.723+0000 7fb35919dec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:26:26.887 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:26 smithi100 bash[31715]: audit 2024-07-28T19:26:25.571612+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-07-28T19:26:26.887 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:26 smithi100 bash[31715]: cluster 2024-07-28T19:26:25.571704+0000 mon.a (mon.0) 627 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-07-28T19:26:26.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:26 smithi100 bash[21828]: audit 2024-07-28T19:26:25.571612+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-07-28T19:26:26.888 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:26 smithi100 bash[21828]: cluster 2024-07-28T19:26:25.571704+0000 mon.a (mon.0) 627 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-07-28T19:26:26.909 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:26 smithi138 bash[23952]: audit 2024-07-28T19:26:25.571612+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14150 172.21.15.100:0/1770356968' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-07-28T19:26:26.910 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:26 smithi138 bash[23952]: cluster 2024-07-28T19:26:25.571704+0000 mon.a (mon.0) 627 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-07-28T19:26:27.234 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:26 smithi138 bash[24937]: debug 2024-07-28T19:26:26.911+0000 7f8f0cbbdec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:26:27.234 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:26 smithi138 bash[24937]: debug 2024-07-28T19:26:26.979+0000 7f8f0cbbdec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:26:27.235 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:27 smithi138 bash[24937]: debug 2024-07-28T19:26:27.135+0000 7f8f0cbbdec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:26:27.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:26 smithi100 bash[22098]: debug 2024-07-28T19:26:26.891+0000 7fb35919dec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:26:27.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:26 smithi100 bash[22098]: debug 2024-07-28T19:26:26.951+0000 7fb35919dec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:26:27.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:27 smithi100 bash[22098]: debug 2024-07-28T19:26:27.099+0000 7fb35919dec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:26:27.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:27 smithi100 bash[22098]: debug 2024-07-28T19:26:27.719+0000 7fb35919dec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:26:27.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:27 smithi100 bash[22098]: debug 2024-07-28T19:26:27.787+0000 7fb35919dec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:26:27.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:27 smithi100 bash[22098]: debug 2024-07-28T19:26:27.851+0000 7fb35919dec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:26:28.049 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:27 smithi138 bash[24937]: debug 2024-07-28T19:26:27.759+0000 7f8f0cbbdec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:26:28.049 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:27 smithi138 bash[24937]: debug 2024-07-28T19:26:27.831+0000 7f8f0cbbdec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:26:28.049 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:27 smithi138 bash[24937]: debug 2024-07-28T19:26:27.895+0000 7f8f0cbbdec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:26:28.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:28 smithi138 bash[24937]: debug 2024-07-28T19:26:28.051+0000 7f8f0cbbdec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:26:28.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:28 smithi138 bash[24937]: debug 2024-07-28T19:26:28.123+0000 7f8f0cbbdec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:26:28.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:28 smithi138 bash[24937]: debug 2024-07-28T19:26:28.191+0000 7f8f0cbbdec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:26:28.368 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:28 smithi100 bash[22098]: debug 2024-07-28T19:26:28.007+0000 7fb35919dec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:26:28.368 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:28 smithi100 bash[22098]: debug 2024-07-28T19:26:28.079+0000 7fb35919dec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:26:28.368 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:28 smithi100 bash[22098]: debug 2024-07-28T19:26:28.147+0000 7fb35919dec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:26:28.673 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:28 smithi100 bash[22098]: debug 2024-07-28T19:26:28.371+0000 7fb35919dec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:26:28.674 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:28 smithi100 bash[22098]: debug 2024-07-28T19:26:28.435+0000 7fb35919dec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:26:28.674 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:28 smithi100 bash[22098]: debug 2024-07-28T19:26:28.599+0000 7fb35919dec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:26:28.726 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:28 smithi138 bash[24937]: debug 2024-07-28T19:26:28.419+0000 7f8f0cbbdec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:26:28.726 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:28 smithi138 bash[24937]: debug 2024-07-28T19:26:28.483+0000 7f8f0cbbdec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:26:28.726 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:28 smithi138 bash[24937]: debug 2024-07-28T19:26:28.651+0000 7f8f0cbbdec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:26:28.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:28 smithi100 bash[22098]: debug 2024-07-28T19:26:28.675+0000 7fb35919dec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:26:28.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:28 smithi100 bash[22098]: debug 2024-07-28T19:26:28.903+0000 7fb35919dec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:26:29.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:28 smithi138 bash[24937]: debug 2024-07-28T19:26:28.727+0000 7f8f0cbbdec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:26:29.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:28 smithi138 bash[24937]: debug 2024-07-28T19:26:28.967+0000 7f8f0cbbdec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:26:29.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:29 smithi100 bash[22098]: debug 2024-07-28T19:26:29.063+0000 7fb35919dec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:26:29.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:29 smithi100 bash[22098]: debug 2024-07-28T19:26:29.139+0000 7fb35919dec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:26:29.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:29 smithi100 bash[22098]: debug 2024-07-28T19:26:29.287+0000 7fb35919dec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:26:29.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:29 smithi138 bash[24937]: debug 2024-07-28T19:26:29.127+0000 7f8f0cbbdec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:26:29.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:29 smithi138 bash[24937]: debug 2024-07-28T19:26:29.203+0000 7f8f0cbbdec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:26:29.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:29 smithi138 bash[24937]: debug 2024-07-28T19:26:29.351+0000 7f8f0cbbdec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:26:30.321 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: debug 2024-07-28T19:26:30.059+0000 7fb35919dec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:26:30.321 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: debug 2024-07-28T19:26:30.131+0000 7fb35919dec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:26:30.377 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: debug 2024-07-28T19:26:30.111+0000 7f8f0cbbdec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:26:30.377 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: debug 2024-07-28T19:26:30.183+0000 7f8f0cbbdec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:26:30.704 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: debug 2024-07-28T19:26:30.323+0000 7fb35919dec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:26:30.704 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: debug 2024-07-28T19:26:30.395+0000 7fb35919dec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:26:30.758 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: debug 2024-07-28T19:26:30.379+0000 7f8f0cbbdec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:26:30.758 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: debug 2024-07-28T19:26:30.451+0000 7f8f0cbbdec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:26:30.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: debug 2024-07-28T19:26:30.707+0000 7fb35919dec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:26:30.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: debug 2024-07-28T19:26:30.779+0000 7fb35919dec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:26:30.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: [28/Jul/2024:19:26:30] ENGINE Bus STARTING 2024-07-28T19:26:30.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: CherryPy Checker: 2024-07-28T19:26:30.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[22098]: The Application mounted at '' has an empty config. 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: cluster 2024-07-28T19:26:30.807154+0000 mon.a (mon.0) 628 : cluster [INF] Active manager daemon y restarted 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: cluster 2024-07-28T19:26:30.807809+0000 mon.a (mon.0) 629 : cluster [INF] Activating manager daemon y 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: cluster 2024-07-28T19:26:30.819779+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.820332+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: cluster 2024-07-28T19:26:30.820530+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0129028s), standbys: x 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.820531+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.820660+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.822095+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.822275+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:26:30.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.822456+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.822661+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.822854+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.823048+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.823236+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.823530+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.823819+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.824089+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.824345+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.824517+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: audit 2024-07-28T19:26:30.824986+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:26:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[21828]: cluster 2024-07-28T19:26:30.852068+0000 mon.a (mon.0) 632 : cluster [INF] Manager daemon y is now available 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: cluster 2024-07-28T19:26:30.807154+0000 mon.a (mon.0) 628 : cluster [INF] Active manager daemon y restarted 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: cluster 2024-07-28T19:26:30.807809+0000 mon.a (mon.0) 629 : cluster [INF] Activating manager daemon y 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: cluster 2024-07-28T19:26:30.819779+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.820332+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: cluster 2024-07-28T19:26:30.820530+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0129028s), standbys: x 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.820531+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.820660+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.822095+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.822275+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.822456+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.822661+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:26:30.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.822854+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.823048+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.823236+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.823530+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.823819+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.824089+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.824345+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.824517+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: audit 2024-07-28T19:26:30.824986+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:26:30.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:30 smithi100 bash[31715]: cluster 2024-07-28T19:26:30.852068+0000 mon.a (mon.0) 632 : cluster [INF] Manager daemon y is now available 2024-07-28T19:26:31.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: debug 2024-07-28T19:26:30.759+0000 7f8f0cbbdec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:26:31.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: debug 2024-07-28T19:26:30.839+0000 7f8f0cbbdec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:26:31.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: [28/Jul/2024:19:26:30] ENGINE Bus STARTING 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: CherryPy Checker: 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: The Application mounted at '' has an empty config. 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: [28/Jul/2024:19:26:30] ENGINE Serving on http://:::9283 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[24937]: [28/Jul/2024:19:26:30] ENGINE Bus STARTED 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: cluster 2024-07-28T19:26:30.807154+0000 mon.a (mon.0) 628 : cluster [INF] Active manager daemon y restarted 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: cluster 2024-07-28T19:26:30.807809+0000 mon.a (mon.0) 629 : cluster [INF] Activating manager daemon y 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: cluster 2024-07-28T19:26:30.819779+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.820332+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: cluster 2024-07-28T19:26:30.820530+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0129028s), standbys: x 2024-07-28T19:26:31.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.820531+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.820660+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.822095+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.822275+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.822456+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.822661+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.822854+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.823048+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.823236+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.823530+0000 mon.c (mon.2) 30 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.823819+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.824089+0000 mon.c (mon.2) 32 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.824345+0000 mon.c (mon.2) 33 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.824517+0000 mon.c (mon.2) 34 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:26:31.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: audit 2024-07-28T19:26:30.824986+0000 mon.c (mon.2) 35 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:26:31.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:30 smithi138 bash[23952]: cluster 2024-07-28T19:26:30.852068+0000 mon.a (mon.0) 632 : cluster [INF] Manager daemon y is now available 2024-07-28T19:26:31.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[22098]: [28/Jul/2024:19:26:31] ENGINE Serving on http://:::9283 2024-07-28T19:26:31.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[22098]: [28/Jul/2024:19:26:31] ENGINE Bus STARTED 2024-07-28T19:26:31.845 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled alertmanager update... 2024-07-28T19:26:32.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.863563+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:26:32.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.864339+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.865865+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: cluster 2024-07-28T19:26:30.866489+0000 mon.a (mon.0) 633 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: cluster 2024-07-28T19:26:30.866567+0000 mon.a (mon.0) 634 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.866793+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.871963+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.874031+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.889689+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.902161+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.904882+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.905195+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.913573+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:30.913917+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: cluster 2024-07-28T19:26:31.826586+0000 mon.a (mon.0) 638 : cluster [DBG] mgrmap e18: y(active, since 1.01896s), standbys: x 2024-07-28T19:26:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[31715]: audit 2024-07-28T19:26:31.849723+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.863563+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.864339+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.865865+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: cluster 2024-07-28T19:26:30.866489+0000 mon.a (mon.0) 633 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: cluster 2024-07-28T19:26:30.866567+0000 mon.a (mon.0) 634 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.866793+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.871963+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.874031+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.889689+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:32.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.902161+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:26:32.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.904882+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:26:32.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.905195+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:26:32.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.913573+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:26:32.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:30.913917+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:26:32.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: cluster 2024-07-28T19:26:31.826586+0000 mon.a (mon.0) 638 : cluster [DBG] mgrmap e18: y(active, since 1.01896s), standbys: x 2024-07-28T19:26:32.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:31 smithi100 bash[21828]: audit 2024-07-28T19:26:31.849723+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.863563+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:26:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.864339+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:26:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.865865+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:26:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: cluster 2024-07-28T19:26:30.866489+0000 mon.a (mon.0) 633 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:26:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: cluster 2024-07-28T19:26:30.866567+0000 mon.a (mon.0) 634 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:26:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.866793+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.138:0/778035020' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:26:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.871963+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.874031+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:32.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.889689+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:32.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.902161+0000 mon.c (mon.2) 38 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:26:32.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.904882+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:26:32.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.905195+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:26:32.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.913573+0000 mon.c (mon.2) 40 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:26:32.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:30.913917+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:26:32.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: cluster 2024-07-28T19:26:31.826586+0000 mon.a (mon.0) 638 : cluster [DBG] mgrmap e18: y(active, since 1.01896s), standbys: x 2024-07-28T19:26:32.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:31 smithi138 bash[23952]: audit 2024-07-28T19:26:31.849723+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:32.591 DEBUG:teuthology.orchestra.run.smithi100:alertmanager.a> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@alertmanager.a.service 2024-07-28T19:26:32.593 INFO:tasks.cephadm:Adding grafana.a on smithi138 2024-07-28T19:26:32.594 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph orch apply grafana '1;smithi138=a' 2024-07-28T19:26:32.744 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: debug 2024-07-28T19:26:32.431+0000 7fb2fbf2d640 -1 log_channel(cephadm) log [ERR] : [28/Jul/2024:19:26:32] ENGINE Error in HTTPServer.serve 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: Traceback (most recent call last): 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: self._connections.run(self.expiration_interval) 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: self._run(expiration_interval) 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: s = self.context.wrap_socket( 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:26:32.745 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: return self.sslsocket_class._create( 2024-07-28T19:26:32.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:26:32.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: self.do_handshake() 2024-07-28T19:26:32.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:26:32.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: self._sslobj.do_handshake() 2024-07-28T19:26:32.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:32 smithi100 bash[22098]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: cephadm 2024-07-28T19:26:31.842435+0000 mgr.y (mgr.14391) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi100=a;count:1 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: cluster 2024-07-28T19:26:31.846275+0000 mgr.y (mgr.14391) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: cephadm 2024-07-28T19:26:32.313155+0000 mgr.y (mgr.14391) 4 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Bus STARTING 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: cephadm 2024-07-28T19:26:32.433976+0000 mgr.y (mgr.14391) 5 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: cephadm 2024-07-28T19:26:32.434060+0000 mgr.y (mgr.14391) 6 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Bus STARTED 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: cephadm 2024-07-28T19:26:32.436083+0000 mgr.y (mgr.14391) 7 : cephadm [ERR] [28/Jul/2024:19:26:32] ENGINE Error in HTTPServer.serve 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: Traceback (most recent call last): 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:26:33.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: self._connections.run(self.expiration_interval) 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: self._run(expiration_interval) 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: s = self.context.wrap_socket( 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: return self.sslsocket_class._create( 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:26:33.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: self.do_handshake() 2024-07-28T19:26:33.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:26:33.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: self._sslobj.do_handshake() 2024-07-28T19:26:33.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:26:33.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[21828]: audit 2024-07-28T19:26:32.442438+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:33.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: cephadm 2024-07-28T19:26:31.842435+0000 mgr.y (mgr.14391) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi100=a;count:1 2024-07-28T19:26:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: cluster 2024-07-28T19:26:31.846275+0000 mgr.y (mgr.14391) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: cephadm 2024-07-28T19:26:32.313155+0000 mgr.y (mgr.14391) 4 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Bus STARTING 2024-07-28T19:26:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: cephadm 2024-07-28T19:26:32.433976+0000 mgr.y (mgr.14391) 5 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:26:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: cephadm 2024-07-28T19:26:32.434060+0000 mgr.y (mgr.14391) 6 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Bus STARTED 2024-07-28T19:26:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: cephadm 2024-07-28T19:26:32.436083+0000 mgr.y (mgr.14391) 7 : cephadm [ERR] [28/Jul/2024:19:26:32] ENGINE Error in HTTPServer.serve 2024-07-28T19:26:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: Traceback (most recent call last): 2024-07-28T19:26:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:26:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: self._connections.run(self.expiration_interval) 2024-07-28T19:26:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:26:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: self._run(expiration_interval) 2024-07-28T19:26:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:26:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:26:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: s = self.context.wrap_socket( 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: return self.sslsocket_class._create( 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: self.do_handshake() 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: self._sslobj.do_handshake() 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:26:33.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:33 smithi100 bash[31715]: audit 2024-07-28T19:26:32.442438+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:33.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: cephadm 2024-07-28T19:26:31.842435+0000 mgr.y (mgr.14391) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi100=a;count:1 2024-07-28T19:26:33.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: cluster 2024-07-28T19:26:31.846275+0000 mgr.y (mgr.14391) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: cephadm 2024-07-28T19:26:32.313155+0000 mgr.y (mgr.14391) 4 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Bus STARTING 2024-07-28T19:26:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: cephadm 2024-07-28T19:26:32.433976+0000 mgr.y (mgr.14391) 5 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:26:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: cephadm 2024-07-28T19:26:32.434060+0000 mgr.y (mgr.14391) 6 : cephadm [INF] [28/Jul/2024:19:26:32] ENGINE Bus STARTED 2024-07-28T19:26:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: cephadm 2024-07-28T19:26:32.436083+0000 mgr.y (mgr.14391) 7 : cephadm [ERR] [28/Jul/2024:19:26:32] ENGINE Error in HTTPServer.serve 2024-07-28T19:26:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: Traceback (most recent call last): 2024-07-28T19:26:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:26:33.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: self._connections.run(self.expiration_interval) 2024-07-28T19:26:33.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:26:33.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: self._run(expiration_interval) 2024-07-28T19:26:33.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:26:33.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:26:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:26:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:26:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:26:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: s = self.context.wrap_socket( 2024-07-28T19:26:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:26:33.809 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: return self.sslsocket_class._create( 2024-07-28T19:26:33.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:26:33.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: self.do_handshake() 2024-07-28T19:26:33.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:26:33.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: self._sslobj.do_handshake() 2024-07-28T19:26:33.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:26:33.811 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:33 smithi138 bash[23952]: audit 2024-07-28T19:26:32.442438+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:34.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:34 smithi100 bash[31715]: cluster 2024-07-28T19:26:32.823994+0000 mgr.y (mgr.14391) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:34.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:34 smithi100 bash[31715]: cluster 2024-07-28T19:26:33.461908+0000 mon.a (mon.0) 641 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-07-28T19:26:34.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:34 smithi100 bash[21828]: cluster 2024-07-28T19:26:32.823994+0000 mgr.y (mgr.14391) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:34.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:34 smithi100 bash[21828]: cluster 2024-07-28T19:26:33.461908+0000 mon.a (mon.0) 641 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-07-28T19:26:34.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:34 smithi138 bash[23952]: cluster 2024-07-28T19:26:32.823994+0000 mgr.y (mgr.14391) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:34.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:34 smithi138 bash[23952]: cluster 2024-07-28T19:26:33.461908+0000 mon.a (mon.0) 641 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-07-28T19:26:36.458 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:36 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:26:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:26:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:36 smithi100 bash[21828]: cluster 2024-07-28T19:26:34.824677+0000 mgr.y (mgr.14391) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:36 smithi100 bash[21828]: cluster 2024-07-28T19:26:35.473481+0000 mon.a (mon.0) 642 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-07-28T19:26:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:36 smithi100 bash[21828]: audit 2024-07-28T19:26:36.324671+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:36 smithi100 bash[21828]: audit 2024-07-28T19:26:36.329955+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:36.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:36 smithi100 bash[31715]: cluster 2024-07-28T19:26:34.824677+0000 mgr.y (mgr.14391) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:36.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:36 smithi100 bash[31715]: cluster 2024-07-28T19:26:35.473481+0000 mon.a (mon.0) 642 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-07-28T19:26:36.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:36 smithi100 bash[31715]: audit 2024-07-28T19:26:36.324671+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:36 smithi100 bash[31715]: audit 2024-07-28T19:26:36.329955+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:36.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:36 smithi138 bash[23952]: cluster 2024-07-28T19:26:34.824677+0000 mgr.y (mgr.14391) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:36.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:36 smithi138 bash[23952]: cluster 2024-07-28T19:26:35.473481+0000 mon.a (mon.0) 642 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-07-28T19:26:36.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:36 smithi138 bash[23952]: audit 2024-07-28T19:26:36.324671+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:36.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:36 smithi138 bash[23952]: audit 2024-07-28T19:26:36.329955+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.127 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:26:37.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:36.548242+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:36.560144+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:36.988676+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.000716+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.202861+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.214264+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.217999+0000 mon.c (mon.2) 41 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.218658+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.220864+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.221525+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.223706+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.224331+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.226500+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.227139+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.237747+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.240647+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:37.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:37 smithi138 bash[23952]: audit 2024-07-28T19:26:37.242379+0000 mon.c (mon.2) 46 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:36.548242+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:36.560144+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:36.988676+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.000716+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.202861+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.214264+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.217999+0000 mon.c (mon.2) 41 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.218658+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.220864+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.221525+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.223706+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.224331+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.226500+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.227139+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.237747+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.240647+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[31715]: audit 2024-07-28T19:26:37.242379+0000 mon.c (mon.2) 46 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:36.548242+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:36.560144+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:36.988676+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.000716+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.202861+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.214264+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.217999+0000 mon.c (mon.2) 41 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.218658+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.220864+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.221525+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.223706+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.224331+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.226500+0000 mon.c (mon.2) 44 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.227139+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.237747+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.240647+0000 mon.c (mon.2) 45 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:37.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:37 smithi100 bash[21828]: audit 2024-07-28T19:26:37.242379+0000 mon.c (mon.2) 46 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:38.359 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.359 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.360 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.360 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.360 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.360 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.361 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.361 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.628 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.628 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.628 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: Started Ceph node-exporter.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:26:38.628 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.629 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.629 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.629 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: cluster 2024-07-28T19:26:36.825393+0000 mgr.y (mgr.14391) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:38.629 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.228788+0000 mgr.y (mgr.14391) 11 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 4060M 2024-07-28T19:26:38.629 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.244904+0000 mgr.y (mgr.14391) 12 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:26:38.629 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.245485+0000 mgr.y (mgr.14391) 13 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:26:38.629 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.397599+0000 mgr.y (mgr.14391) 14 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:26:38.629 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.510916+0000 mgr.y (mgr.14391) 15 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:26:38.629 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.553395+0000 mgr.y (mgr.14391) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: audit 2024-07-28T19:26:37.751007+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: audit 2024-07-28T19:26:37.761025+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: audit 2024-07-28T19:26:37.895690+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: audit 2024-07-28T19:26:37.904755+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[21828]: audit 2024-07-28T19:26:37.918123+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: cluster 2024-07-28T19:26:36.825393+0000 mgr.y (mgr.14391) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.228788+0000 mgr.y (mgr.14391) 11 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 4060M 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.244904+0000 mgr.y (mgr.14391) 12 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:26:38.630 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.245485+0000 mgr.y (mgr.14391) 13 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:26:38.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.397599+0000 mgr.y (mgr.14391) 14 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:26:38.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.510916+0000 mgr.y (mgr.14391) 15 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:26:38.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.553395+0000 mgr.y (mgr.14391) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:26:38.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: audit 2024-07-28T19:26:37.751007+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: audit 2024-07-28T19:26:37.761025+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: audit 2024-07-28T19:26:37.895690+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: audit 2024-07-28T19:26:37.904755+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[31715]: audit 2024-07-28T19:26:37.918123+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.631 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.631 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:26:38 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: cluster 2024-07-28T19:26:36.825393+0000 mgr.y (mgr.14391) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.228788+0000 mgr.y (mgr.14391) 11 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 4060M 2024-07-28T19:26:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.244904+0000 mgr.y (mgr.14391) 12 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:26:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.245485+0000 mgr.y (mgr.14391) 13 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:26:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.397599+0000 mgr.y (mgr.14391) 14 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:26:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.510916+0000 mgr.y (mgr.14391) 15 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:26:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.553395+0000 mgr.y (mgr.14391) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:26:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: audit 2024-07-28T19:26:37.751007+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: audit 2024-07-28T19:26:37.761025+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: audit 2024-07-28T19:26:37.895690+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: audit 2024-07-28T19:26:37.904755+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 bash[23952]: audit 2024-07-28T19:26:37.918123+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:38.994 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:38 smithi100 bash[56136]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-07-28T19:26:39.159 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:38 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.159 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:38 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.159 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:26:38 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.160 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:38 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.160 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:26:38 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.160 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:26:38 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.161 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:26:38 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.469 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: v1.5.0: Pulling from prometheus/node-exporter 2024-07-28T19:26:39.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:39 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.554 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:26:39 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.555 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:26:39 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.555 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:26:39 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.555 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:26:39 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[48987]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-07-28T19:26:39.556 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:26:39 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.723 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: 22b70bddd3ac: Pulling fs layer 2024-07-28T19:26:39.723 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: 5c12815fee55: Pulling fs layer 2024-07-28T19:26:39.723 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: c0e87333d380: Pulling fs layer 2024-07-28T19:26:39.723 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: 5c12815fee55: Download complete 2024-07-28T19:26:39.723 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: 22b70bddd3ac: Download complete 2024-07-28T19:26:39.723 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: c0e87333d380: Verifying Checksum 2024-07-28T19:26:39.724 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:39.724 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.628441+0000 mgr.y (mgr.14391) 17 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:26:39.724 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.683304+0000 mgr.y (mgr.14391) 18 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:26:39.724 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.794497+0000 mgr.y (mgr.14391) 19 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:26:39.724 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: cephadm 2024-07-28T19:26:37.926229+0000 mgr.y (mgr.14391) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi100 2024-07-28T19:26:39.724 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: audit 2024-07-28T19:26:38.628661+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: audit 2024-07-28T19:26:38.636680+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: audit 2024-07-28T19:26:38.641053+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: audit 2024-07-28T19:26:39.306053+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: audit 2024-07-28T19:26:39.317175+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: audit 2024-07-28T19:26:39.327623+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: audit 2024-07-28T19:26:39.335756+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[31715]: audit 2024-07-28T19:26:39.351929+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.628441+0000 mgr.y (mgr.14391) 17 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.683304+0000 mgr.y (mgr.14391) 18 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.794497+0000 mgr.y (mgr.14391) 19 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: cephadm 2024-07-28T19:26:37.926229+0000 mgr.y (mgr.14391) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi100 2024-07-28T19:26:39.725 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: audit 2024-07-28T19:26:38.628661+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.726 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: audit 2024-07-28T19:26:38.636680+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.726 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: audit 2024-07-28T19:26:38.641053+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.726 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: audit 2024-07-28T19:26:39.306053+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.726 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: audit 2024-07-28T19:26:39.317175+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.726 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: audit 2024-07-28T19:26:39.327623+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.726 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: audit 2024-07-28T19:26:39.335756+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.726 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[21828]: audit 2024-07-28T19:26:39.351929+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.628441+0000 mgr.y (mgr.14391) 17 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.683304+0000 mgr.y (mgr.14391) 18 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.794497+0000 mgr.y (mgr.14391) 19 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: cephadm 2024-07-28T19:26:37.926229+0000 mgr.y (mgr.14391) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi100 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: audit 2024-07-28T19:26:38.628661+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: audit 2024-07-28T19:26:38.636680+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: audit 2024-07-28T19:26:38.641053+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: audit 2024-07-28T19:26:39.306053+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: audit 2024-07-28T19:26:39.317175+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: audit 2024-07-28T19:26:39.327623+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: audit 2024-07-28T19:26:39.335756+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:39 smithi138 bash[23952]: audit 2024-07-28T19:26:39.351929+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:39.994 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: c0e87333d380: Download complete 2024-07-28T19:26:39.994 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:39 smithi100 bash[56136]: 22b70bddd3ac: Pull complete 2024-07-28T19:26:40.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: v1.5.0: Pulling from prometheus/node-exporter 2024-07-28T19:26:40.743 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:40 smithi100 bash[56136]: 5c12815fee55: Pull complete 2024-07-28T19:26:40.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:40 smithi100 bash[31715]: cephadm 2024-07-28T19:26:38.641815+0000 mgr.y (mgr.14391) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi138 2024-07-28T19:26:40.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:40 smithi100 bash[31715]: cluster 2024-07-28T19:26:38.825996+0000 mgr.y (mgr.14391) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-28T19:26:40.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:40 smithi100 bash[31715]: cephadm 2024-07-28T19:26:39.377074+0000 mgr.y (mgr.14391) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi100 2024-07-28T19:26:40.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:40 smithi100 bash[21828]: cephadm 2024-07-28T19:26:38.641815+0000 mgr.y (mgr.14391) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi138 2024-07-28T19:26:40.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:40 smithi100 bash[21828]: cluster 2024-07-28T19:26:38.825996+0000 mgr.y (mgr.14391) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-28T19:26:40.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:40 smithi100 bash[21828]: cephadm 2024-07-28T19:26:39.377074+0000 mgr.y (mgr.14391) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi100 2024-07-28T19:26:40.851 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[23952]: cephadm 2024-07-28T19:26:38.641815+0000 mgr.y (mgr.14391) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi138 2024-07-28T19:26:40.851 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[23952]: cluster 2024-07-28T19:26:38.825996+0000 mgr.y (mgr.14391) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-07-28T19:26:40.851 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[23952]: cephadm 2024-07-28T19:26:39.377074+0000 mgr.y (mgr.14391) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi100 2024-07-28T19:26:40.852 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: 22b70bddd3ac: Pulling fs layer 2024-07-28T19:26:40.852 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: 5c12815fee55: Pulling fs layer 2024-07-28T19:26:40.852 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: c0e87333d380: Pulling fs layer 2024-07-28T19:26:40.852 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: 22b70bddd3ac: Download complete 2024-07-28T19:26:40.852 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: 5c12815fee55: Verifying Checksum 2024-07-28T19:26:40.852 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: 5c12815fee55: Download complete 2024-07-28T19:26:40.852 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: c0e87333d380: Verifying Checksum 2024-07-28T19:26:40.852 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: c0e87333d380: Download complete 2024-07-28T19:26:41.047 INFO:teuthology.orchestra.run.smithi138.stdout:Scheduled grafana update... 2024-07-28T19:26:41.243 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:40 smithi100 bash[56136]: c0e87333d380: Pull complete 2024-07-28T19:26:41.305 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:40 smithi138 bash[48987]: 22b70bddd3ac: Pull complete 2024-07-28T19:26:41.744 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[56136]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-07-28T19:26:41.744 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[56136]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-07-28T19:26:42.133 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:41 smithi138 bash[48987]: 5c12815fee55: Pull complete 2024-07-28T19:26:42.133 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:41 smithi138 bash[23952]: cluster 2024-07-28T19:26:40.826386+0000 mgr.y (mgr.14391) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-28T19:26:42.133 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:41 smithi138 bash[23952]: audit 2024-07-28T19:26:40.913637+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:42.133 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:41 smithi138 bash[23952]: audit 2024-07-28T19:26:41.040792+0000 mgr.y (mgr.14391) 25 : audit [DBG] from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:42.133 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:41 smithi138 bash[23952]: cephadm 2024-07-28T19:26:41.043572+0000 mgr.y (mgr.14391) 26 : cephadm [INF] Saving service grafana spec with placement smithi138=a;count:1 2024-07-28T19:26:42.133 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:41 smithi138 bash[23952]: audit 2024-07-28T19:26:41.052674+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:42.134 DEBUG:teuthology.orchestra.run.smithi138:grafana.a> sudo journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@grafana.a.service 2024-07-28T19:26:42.136 INFO:tasks.cephadm:Setting up client nodes... 2024-07-28T19:26:42.137 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-07-28T19:26:42.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[31715]: cluster 2024-07-28T19:26:40.826386+0000 mgr.y (mgr.14391) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-28T19:26:42.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[31715]: audit 2024-07-28T19:26:40.913637+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:42.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[31715]: audit 2024-07-28T19:26:41.040792+0000 mgr.y (mgr.14391) 25 : audit [DBG] from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:42.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[31715]: cephadm 2024-07-28T19:26:41.043572+0000 mgr.y (mgr.14391) 26 : cephadm [INF] Saving service grafana spec with placement smithi138=a;count:1 2024-07-28T19:26:42.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[31715]: audit 2024-07-28T19:26:41.052674+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[21828]: cluster 2024-07-28T19:26:40.826386+0000 mgr.y (mgr.14391) 24 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-07-28T19:26:42.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[21828]: audit 2024-07-28T19:26:40.913637+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:42.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[21828]: audit 2024-07-28T19:26:41.040792+0000 mgr.y (mgr.14391) 25 : audit [DBG] from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:26:42.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[21828]: cephadm 2024-07-28T19:26:41.043572+0000 mgr.y (mgr.14391) 26 : cephadm [INF] Saving service grafana spec with placement smithi138=a;count:1 2024-07-28T19:26:42.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:41 smithi100 bash[21828]: audit 2024-07-28T19:26:41.052674+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:42.767 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:42 smithi138 bash[48987]: c0e87333d380: Pull complete 2024-07-28T19:26:42.767 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:42 smithi138 bash[48987]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-07-28T19:26:42.767 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:42 smithi138 bash[48987]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-07-28T19:26:44.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:43 smithi100 bash[21828]: cluster 2024-07-28T19:26:42.826997+0000 mgr.y (mgr.14391) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-28T19:26:44.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:43 smithi100 bash[31715]: cluster 2024-07-28T19:26:42.826997+0000 mgr.y (mgr.14391) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-28T19:26:44.244 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.096Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.096Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=arp 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=bcache 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=bonding 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=btrfs 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=conntrack 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=cpu 2024-07-28T19:26:44.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=diskstats 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=dmi 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.097Z caller=node_exporter.go:117 level=info collector=edac 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=entropy 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=filefd 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=filesystem 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=hwmon 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=infiniband 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=ipvs 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=loadavg 2024-07-28T19:26:44.246 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=mdadm 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=meminfo 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=netclass 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=netdev 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=netstat 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=nfs 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=nfsd 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=nvme 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=os 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-07-28T19:26:44.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=pressure 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=rapl 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=schedstat 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=selinux 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=sockstat 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=softnet 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=stat 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=tapestats 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=textfile 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=time 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=uname 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=vmstat 2024-07-28T19:26:44.248 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=xfs 2024-07-28T19:26:44.249 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=node_exporter.go:117 level=info collector=zfs 2024-07-28T19:26:44.249 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-07-28T19:26:44.249 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:44 smithi100 bash[56136]: ts=2024-07-28T19:26:44.098Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-07-28T19:26:44.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:43 smithi138 bash[23952]: cluster 2024-07-28T19:26:42.826997+0000 mgr.y (mgr.14391) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.034Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.034Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.035Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.035Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.035Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.035Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=arp 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=bcache 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=bonding 2024-07-28T19:26:44.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=btrfs 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=conntrack 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=cpu 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=diskstats 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=dmi 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=edac 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=entropy 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=filefd 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=filesystem 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=hwmon 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=infiniband 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=ipvs 2024-07-28T19:26:44.307 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=loadavg 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=mdadm 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=meminfo 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=netclass 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=netdev 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=netstat 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=nfs 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=nfsd 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=nvme 2024-07-28T19:26:44.308 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=os 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=pressure 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=rapl 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=schedstat 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=selinux 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=sockstat 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=softnet 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=stat 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=tapestats 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=textfile 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=time 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=uname 2024-07-28T19:26:44.309 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=vmstat 2024-07-28T19:26:44.310 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=xfs 2024-07-28T19:26:44.310 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.036Z caller=node_exporter.go:117 level=info collector=zfs 2024-07-28T19:26:44.310 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.037Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-07-28T19:26:44.310 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:26:44 smithi138 bash[48987]: ts=2024-07-28T19:26:44.037Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-07-28T19:26:46.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:45 smithi100 bash[21828]: cluster 2024-07-28T19:26:44.827582+0000 mgr.y (mgr.14391) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:46.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:45 smithi100 bash[21828]: audit 2024-07-28T19:26:45.902654+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:26:46.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:45 smithi100 bash[31715]: cluster 2024-07-28T19:26:44.827582+0000 mgr.y (mgr.14391) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:46.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:45 smithi100 bash[31715]: audit 2024-07-28T19:26:45.902654+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:26:46.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:46 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:26:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:26:46.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:45 smithi138 bash[23952]: cluster 2024-07-28T19:26:44.827582+0000 mgr.y (mgr.14391) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:46.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:45 smithi138 bash[23952]: audit 2024-07-28T19:26:45.902654+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:26:47.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:47 smithi100 bash[21828]: cluster 2024-07-28T19:26:46.828182+0000 mgr.y (mgr.14391) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:47.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:47 smithi100 bash[31715]: cluster 2024-07-28T19:26:46.828182+0000 mgr.y (mgr.14391) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:48.058 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:26:48.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:47 smithi138 bash[23952]: cluster 2024-07-28T19:26:46.828182+0000 mgr.y (mgr.14391) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:50.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:49 smithi100 bash[21828]: cluster 2024-07-28T19:26:48.828933+0000 mgr.y (mgr.14391) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:50.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:49 smithi100 bash[31715]: cluster 2024-07-28T19:26:48.828933+0000 mgr.y (mgr.14391) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:50.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:49 smithi138 bash[23952]: cluster 2024-07-28T19:26:48.828933+0000 mgr.y (mgr.14391) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-07-28T19:26:50.837 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:50.837 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:50.838 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:50.838 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:50.838 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:50.839 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:50.839 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:50.839 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:50.839 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.244 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.244 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.245 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.245 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.245 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.246 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.246 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.246 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:50 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:51.246 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:51 smithi100 systemd[1]: Started Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:26:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[21828]: cluster 2024-07-28T19:26:50.829601+0000 mgr.y (mgr.14391) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[21828]: audit 2024-07-28T19:26:51.118358+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[21828]: audit 2024-07-28T19:26:51.124568+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[21828]: audit 2024-07-28T19:26:51.129455+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[21828]: audit 2024-07-28T19:26:51.133744+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[21828]: audit 2024-07-28T19:26:51.162925+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[21828]: audit 2024-07-28T19:26:51.400809+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[21828]: audit 2024-07-28T19:26:51.405827+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[31715]: cluster 2024-07-28T19:26:50.829601+0000 mgr.y (mgr.14391) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[31715]: audit 2024-07-28T19:26:51.118358+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[31715]: audit 2024-07-28T19:26:51.124568+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[31715]: audit 2024-07-28T19:26:51.129455+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[31715]: audit 2024-07-28T19:26:51.133744+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[31715]: audit 2024-07-28T19:26:51.162925+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[31715]: audit 2024-07-28T19:26:51.400809+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[31715]: audit 2024-07-28T19:26:51.405827+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:52 smithi138 bash[23952]: cluster 2024-07-28T19:26:50.829601+0000 mgr.y (mgr.14391) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:52.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:52 smithi138 bash[23952]: audit 2024-07-28T19:26:51.118358+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:52 smithi138 bash[23952]: audit 2024-07-28T19:26:51.124568+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:52 smithi138 bash[23952]: audit 2024-07-28T19:26:51.129455+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:52 smithi138 bash[23952]: audit 2024-07-28T19:26:51.133744+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:52 smithi138 bash[23952]: audit 2024-07-28T19:26:51.162925+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:26:52.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:52 smithi138 bash[23952]: audit 2024-07-28T19:26:51.400809+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:52 smithi138 bash[23952]: audit 2024-07-28T19:26:51.405827+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:52.944 INFO:teuthology.orchestra.run.smithi100.stdout:[client.0] 2024-07-28T19:26:52.945 INFO:teuthology.orchestra.run.smithi100.stdout: key = AQB8m6Zm5txSOBAAmQF+458Ozf36GhfBKdAd9g== 2024-07-28T19:26:53.122 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[56598]: ts=2024-07-28T19:26:52.859Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-07-28T19:26:53.122 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[56598]: ts=2024-07-28T19:26:52.859Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-07-28T19:26:53.122 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[56598]: ts=2024-07-28T19:26:52.860Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.100 port=9094 2024-07-28T19:26:53.122 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[56598]: ts=2024-07-28T19:26:52.861Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-07-28T19:26:53.122 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[56598]: ts=2024-07-28T19:26:52.894Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-28T19:26:53.122 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[56598]: ts=2024-07-28T19:26:52.894Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-28T19:26:53.122 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[56598]: ts=2024-07-28T19:26:52.897Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-07-28T19:26:53.123 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:52 smithi100 bash[56598]: ts=2024-07-28T19:26:52.897Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-07-28T19:26:53.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:53 smithi100 bash[21828]: audit 2024-07-28T19:26:52.944144+0000 mon.c (mon.2) 49 : audit [INF] from='client.? 172.21.15.100:0/4032388342' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:26:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:53 smithi100 bash[21828]: audit 2024-07-28T19:26:52.944657+0000 mon.a (mon.0) 677 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:26:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:53 smithi100 bash[21828]: audit 2024-07-28T19:26:52.951026+0000 mon.a (mon.0) 678 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-28T19:26:53.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:53 smithi100 bash[31715]: audit 2024-07-28T19:26:52.944144+0000 mon.c (mon.2) 49 : audit [INF] from='client.? 172.21.15.100:0/4032388342' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:26:53.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:53 smithi100 bash[31715]: audit 2024-07-28T19:26:52.944657+0000 mon.a (mon.0) 677 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:26:53.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:53 smithi100 bash[31715]: audit 2024-07-28T19:26:52.951026+0000 mon.a (mon.0) 678 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-28T19:26:53.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:53 smithi138 bash[23952]: audit 2024-07-28T19:26:52.944144+0000 mon.c (mon.2) 49 : audit [INF] from='client.? 172.21.15.100:0/4032388342' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:26:53.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:53 smithi138 bash[23952]: audit 2024-07-28T19:26:52.944657+0000 mon.a (mon.0) 677 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:26:53.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:53 smithi138 bash[23952]: audit 2024-07-28T19:26:52.951026+0000 mon.a (mon.0) 678 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-28T19:26:53.672 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:26:53.672 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-07-28T19:26:53.672 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-07-28T19:26:53.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-07-28T19:26:54.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:54 smithi100 bash[21828]: cluster 2024-07-28T19:26:52.830006+0000 mgr.y (mgr.14391) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:54.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:54 smithi100 bash[31715]: cluster 2024-07-28T19:26:52.830006+0000 mgr.y (mgr.14391) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:54.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:54 smithi138 bash[23952]: cluster 2024-07-28T19:26:52.830006+0000 mgr.y (mgr.14391) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:55.243 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:26:54 smithi100 bash[56598]: ts=2024-07-28T19:26:54.862Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001057066s 2024-07-28T19:26:56.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:56 smithi100 bash[31715]: cluster 2024-07-28T19:26:54.830668+0000 mgr.y (mgr.14391) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:56.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:56 smithi100 bash[31715]: audit 2024-07-28T19:26:55.924171+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:56.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:56 smithi100 bash[21828]: cluster 2024-07-28T19:26:54.830668+0000 mgr.y (mgr.14391) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:56.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:56 smithi100 bash[21828]: audit 2024-07-28T19:26:55.924171+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:56.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:56 smithi138 bash[23952]: cluster 2024-07-28T19:26:54.830668+0000 mgr.y (mgr.14391) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:56.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:56 smithi138 bash[23952]: audit 2024-07-28T19:26:55.924171+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:56.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:26:56 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:26:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:26:57.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:57 smithi100 bash[21828]: audit 2024-07-28T19:26:56.459573+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:57.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:57 smithi100 bash[21828]: audit 2024-07-28T19:26:56.471921+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:57.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:57 smithi100 bash[31715]: audit 2024-07-28T19:26:56.459573+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:57.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:57 smithi100 bash[31715]: audit 2024-07-28T19:26:56.471921+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:57.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:57 smithi138 bash[23952]: audit 2024-07-28T19:26:56.459573+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:57.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:57 smithi138 bash[23952]: audit 2024-07-28T19:26:56.471921+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:57.976 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.b/config 2024-07-28T19:26:58.172 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:57 smithi100 bash[22098]: [28/Jul/2024:19:26:57] ENGINE Bus STOPPING 2024-07-28T19:26:58.172 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[22098]: [28/Jul/2024:19:26:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:26:58.172 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[22098]: [28/Jul/2024:19:26:58] ENGINE Bus STOPPED 2024-07-28T19:26:58.172 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[22098]: [28/Jul/2024:19:26:58] ENGINE Bus STARTING 2024-07-28T19:26:58.172 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[22098]: [28/Jul/2024:19:26:58] ENGINE Serving on http://:::9283 2024-07-28T19:26:58.465 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[22098]: [28/Jul/2024:19:26:58] ENGINE Bus STARTED 2024-07-28T19:26:58.466 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:26:58 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:26:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: cluster 2024-07-28T19:26:56.831273+0000 mgr.y (mgr.14391) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.741827+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.747108+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.748390+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.749180+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.754203+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.836740+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.844103+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.849955+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[21828]: audit 2024-07-28T19:26:57.856437+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: cluster 2024-07-28T19:26:56.831273+0000 mgr.y (mgr.14391) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.741827+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.747108+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.748390+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.749180+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.754203+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.836740+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.844103+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.849955+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-28T19:26:58.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:58 smithi100 bash[31715]: audit 2024-07-28T19:26:57.856437+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: cluster 2024-07-28T19:26:56.831273+0000 mgr.y (mgr.14391) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:26:58.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.741827+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.747108+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.748390+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:26:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.749180+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:26:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.754203+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.836740+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.844103+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.849955+0000 mon.c (mon.2) 52 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-28T19:26:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:58 smithi138 bash[23952]: audit 2024-07-28T19:26:57.856437+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:26:59.743 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:59 smithi100 bash[31715]: audit 2024-07-28T19:26:57.851088+0000 mgr.y (mgr.14391) 35 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-28T19:26:59.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:26:59 smithi100 bash[31715]: cephadm 2024-07-28T19:26:57.876755+0000 mgr.y (mgr.14391) 36 : cephadm [INF] Deploying daemon grafana.a on smithi138 2024-07-28T19:26:59.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:59 smithi100 bash[21828]: audit 2024-07-28T19:26:57.851088+0000 mgr.y (mgr.14391) 35 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-28T19:26:59.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:26:59 smithi100 bash[21828]: cephadm 2024-07-28T19:26:57.876755+0000 mgr.y (mgr.14391) 36 : cephadm [INF] Deploying daemon grafana.a on smithi138 2024-07-28T19:26:59.745 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:59 smithi138 bash[23952]: audit 2024-07-28T19:26:57.851088+0000 mgr.y (mgr.14391) 35 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-07-28T19:26:59.745 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:26:59 smithi138 bash[23952]: cephadm 2024-07-28T19:26:57.876755+0000 mgr.y (mgr.14391) 36 : cephadm [INF] Deploying daemon grafana.a on smithi138 2024-07-28T19:27:00.743 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:00 smithi100 bash[31715]: cluster 2024-07-28T19:26:58.831928+0000 mgr.y (mgr.14391) 37 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:00.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:00 smithi100 bash[21828]: cluster 2024-07-28T19:26:58.831928+0000 mgr.y (mgr.14391) 37 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:00.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:00 smithi138 bash[23952]: cluster 2024-07-28T19:26:58.831928+0000 mgr.y (mgr.14391) 37 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:01.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:01 smithi100 bash[21828]: audit 2024-07-28T19:27:00.903393+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:01.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:01 smithi100 bash[31715]: audit 2024-07-28T19:27:00.903393+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:01.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:01 smithi138 bash[23952]: audit 2024-07-28T19:27:00.903393+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:02.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:02 smithi100 bash[21828]: cluster 2024-07-28T19:27:00.832525+0000 mgr.y (mgr.14391) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:02.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:02 smithi100 bash[31715]: cluster 2024-07-28T19:27:00.832525+0000 mgr.y (mgr.14391) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:02.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:02 smithi138 bash[23952]: cluster 2024-07-28T19:27:00.832525+0000 mgr.y (mgr.14391) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:03.243 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:02 smithi100 bash[56598]: ts=2024-07-28T19:27:02.864Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002997944s 2024-07-28T19:27:03.332 INFO:teuthology.orchestra.run.smithi138.stdout:[client.1] 2024-07-28T19:27:03.332 INFO:teuthology.orchestra.run.smithi138.stdout: key = AQCHm6ZmvjbGExAAc8tShuZ08dS7r2/O1hL0xg== 2024-07-28T19:27:03.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:03 smithi138 bash[23952]: audit 2024-07-28T19:27:03.327872+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.138:0/4152538334' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:27:03.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:03 smithi138 bash[23952]: audit 2024-07-28T19:27:03.331432+0000 mon.a (mon.0) 688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:27:03.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:03 smithi138 bash[23952]: audit 2024-07-28T19:27:03.338788+0000 mon.a (mon.0) 689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-28T19:27:03.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:03 smithi100 bash[21828]: audit 2024-07-28T19:27:03.327872+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.138:0/4152538334' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:27:03.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:03 smithi100 bash[21828]: audit 2024-07-28T19:27:03.331432+0000 mon.a (mon.0) 688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:27:03.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:03 smithi100 bash[21828]: audit 2024-07-28T19:27:03.338788+0000 mon.a (mon.0) 689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-28T19:27:03.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:03 smithi100 bash[31715]: audit 2024-07-28T19:27:03.327872+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.138:0/4152538334' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:27:03.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:03 smithi100 bash[31715]: audit 2024-07-28T19:27:03.331432+0000 mon.a (mon.0) 688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-07-28T19:27:03.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:03 smithi100 bash[31715]: audit 2024-07-28T19:27:03.338788+0000 mon.a (mon.0) 689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-07-28T19:27:04.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:04 smithi138 bash[23952]: cluster 2024-07-28T19:27:02.833171+0000 mgr.y (mgr.14391) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:04.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:04 smithi100 bash[21828]: cluster 2024-07-28T19:27:02.833171+0000 mgr.y (mgr.14391) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:04.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:04 smithi100 bash[31715]: cluster 2024-07-28T19:27:02.833171+0000 mgr.y (mgr.14391) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:05.440 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-07-28T19:27:05.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-07-28T19:27:05.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-07-28T19:27:05.462 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-07-28T19:27:05.462 DEBUG:teuthology.orchestra.run.smithi100:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-07-28T19:27:05.756 INFO:teuthology.misc.health.smithi100.stdout: 2024-07-28T19:27:05.757 INFO:teuthology.misc.health.smithi100.stdout:{"epoch":60,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","created":"2024-07-28T19:16:00.842363+0000","modified":"2024-07-28T19:26:30.807300+0000","last_up_change":"2024-07-28T19:25:33.970932+0000","last_in_change":"2024-07-28T19:25:08.436673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-28T19:21:19.886730+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"c4a49140-8022-42e2-bba9-333eb0e88545","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6803","nonce":2528423215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6805","nonce":2528423215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6809","nonce":2528423215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6807","nonce":2528423215}]},"public_addr":"172.21.15.100:6803/2528423215","cluster_addr":"172.21.15.100:6805/2528423215","heartbeat_back_addr":"172.21.15.100:6809/2528423215","heartbeat_front_addr":"172.21.15.100:6807/2528423215","state":["exists","up"]},{"osd":1,"uuid":"4117fbb5-c88e-4d12-bc76-77050b2c0425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6811","nonce":3874116596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6813","nonce":3874116596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6817","nonce":3874116596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6815","nonce":3874116596}]},"public_addr":"172.21.15.100:6811/3874116596","cluster_addr":"172.21.15.100:6813/3874116596","heartbeat_back_addr":"172.21.15.100:6817/3874116596","heartbeat_front_addr":"172.21.15.100:6815/3874116596","state":["exists","up"]},{"osd":2,"uuid":"f3416396-83c7-42fb-9153-9cb806281347","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6819","nonce":2139838255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6821","nonce":2139838255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6825","nonce":2139838255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6823","nonce":2139838255}]},"public_addr":"172.21.15.100:6819/2139838255","cluster_addr":"172.21.15.100:6821/2139838255","heartbeat_back_addr":"172.21.15.100:6825/2139838255","heartbeat_front_addr":"172.21.15.100:6823/2139838255","state":["exists","up"]},{"osd":3,"uuid":"0363a205-8262-42a4-aaaf-e52558fec7f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6827","nonce":684533287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6829","nonce":684533287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6832","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6833","nonce":684533287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6831","nonce":684533287}]},"public_addr":"172.21.15.100:6827/684533287","cluster_addr":"172.21.15.100:6829/684533287","heartbeat_back_addr":"172.21.15.100:6833/684533287","heartbeat_front_addr":"172.21.15.100:6831/684533287","state":["exists","up"]},{"osd":4,"uuid":"75f51136-e1cc-4d3a-89d8-1618aaff366e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6801","nonce":1306402242}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6803","nonce":1306402242}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6807","nonce":1306402242}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6805","nonce":1306402242}]},"public_addr":"172.21.15.138:6801/1306402242","cluster_addr":"172.21.15.138:6803/1306402242","heartbeat_back_addr":"172.21.15.138:6807/1306402242","heartbeat_front_addr":"172.21.15.138:6805/1306402242","state":["exists","up"]},{"osd":5,"uuid":"0a3e4950-a201-4105-9e45-ef45ce2d5f3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6809","nonce":1102784487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6811","nonce":1102784487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6815","nonce":1102784487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6813","nonce":1102784487}]},"public_addr":"172.21.15.138:6809/1102784487","cluster_addr":"172.21.15.138:6811/1102784487","heartbeat_back_addr":"172.21.15.138:6815/1102784487","heartbeat_front_addr":"172.21.15.138:6813/1102784487","state":["exists","up"]},{"osd":6,"uuid":"8a20da48-aee9-44d8-a3db-63f916c6b763","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6817","nonce":3709444988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6819","nonce":3709444988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6823","nonce":3709444988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6821","nonce":3709444988}]},"public_addr":"172.21.15.138:6817/3709444988","cluster_addr":"172.21.15.138:6819/3709444988","heartbeat_back_addr":"172.21.15.138:6823/3709444988","heartbeat_front_addr":"172.21.15.138:6821/3709444988","state":["exists","up"]},{"osd":7,"uuid":"d52b6b70-462f-42ed-966f-fa14921ba172","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6825","nonce":1158882219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6827","nonce":1158882219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6831","nonce":1158882219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6829","nonce":1158882219}]},"public_addr":"172.21.15.138:6825/1158882219","cluster_addr":"172.21.15.138:6827/1158882219","heartbeat_back_addr":"172.21.15.138:6831/1158882219","heartbeat_front_addr":"172.21.15.138:6829/1158882219","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:19:30.916069+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:20:23.352940+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:21:16.853645+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:09.540873+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:59.198788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:23:49.135324+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:24:40.795346+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:25:32.389079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.100:0/638886582":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3154304581":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6801/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/2863303498":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/989116380":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3841450082":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/553075364":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3152621533":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2291421462":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/1006451977":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3944711986":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6800/728100671":"2024-07-29T19:26:30.807202+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-28T19:27:05.775 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-07-28T19:27:05.775 INFO:tasks.cephadm:Creating RBD pool 2024-07-28T19:27:05.775 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-07-28T19:27:06.501 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:27:06 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:27:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:27:06.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:06 smithi138 bash[23952]: cluster 2024-07-28T19:27:04.833747+0000 mgr.y (mgr.14391) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:06.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:06 smithi138 bash[23952]: audit 2024-07-28T19:27:05.762497+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.100:0/3473834509' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:06.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:06 smithi100 bash[31715]: cluster 2024-07-28T19:27:04.833747+0000 mgr.y (mgr.14391) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:06.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:06 smithi100 bash[31715]: audit 2024-07-28T19:27:05.762497+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.100:0/3473834509' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:06.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:06 smithi100 bash[21828]: cluster 2024-07-28T19:27:04.833747+0000 mgr.y (mgr.14391) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:06.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:06 smithi100 bash[21828]: audit 2024-07-28T19:27:05.762497+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.100:0/3473834509' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:08.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:08 smithi138 bash[23952]: cluster 2024-07-28T19:27:06.834294+0000 mgr.y (mgr.14391) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:08.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:08 smithi100 bash[21828]: cluster 2024-07-28T19:27:06.834294+0000 mgr.y (mgr.14391) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:08.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:08 smithi100 bash[31715]: cluster 2024-07-28T19:27:06.834294+0000 mgr.y (mgr.14391) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:10.539 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:10.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:10 smithi138 bash[23952]: cluster 2024-07-28T19:27:08.834859+0000 mgr.y (mgr.14391) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:10.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:10 smithi100 bash[21828]: cluster 2024-07-28T19:27:08.834859+0000 mgr.y (mgr.14391) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:10.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:10 smithi100 bash[31715]: cluster 2024-07-28T19:27:08.834859+0000 mgr.y (mgr.14391) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:12.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:12 smithi138 bash[23952]: cluster 2024-07-28T19:27:10.835466+0000 mgr.y (mgr.14391) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:12.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:12 smithi100 bash[21828]: cluster 2024-07-28T19:27:10.835466+0000 mgr.y (mgr.14391) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:12.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:12 smithi100 bash[31715]: cluster 2024-07-28T19:27:10.835466+0000 mgr.y (mgr.14391) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:14.538 INFO:teuthology.orchestra.run.smithi100.stderr:pool 'rbd' created 2024-07-28T19:27:14.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:14 smithi138 bash[23952]: cluster 2024-07-28T19:27:12.836045+0000 mgr.y (mgr.14391) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:14.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:14 smithi138 bash[23952]: audit 2024-07-28T19:27:13.963372+0000 mon.c (mon.2) 55 : audit [INF] from='client.? 172.21.15.100:0/270232258' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-28T19:27:14.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:14 smithi138 bash[23952]: audit 2024-07-28T19:27:13.964159+0000 mon.a (mon.0) 690 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-28T19:27:14.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:14 smithi100 bash[21828]: cluster 2024-07-28T19:27:12.836045+0000 mgr.y (mgr.14391) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:14.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:14 smithi100 bash[21828]: audit 2024-07-28T19:27:13.963372+0000 mon.c (mon.2) 55 : audit [INF] from='client.? 172.21.15.100:0/270232258' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-28T19:27:14.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:14 smithi100 bash[21828]: audit 2024-07-28T19:27:13.964159+0000 mon.a (mon.0) 690 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-28T19:27:14.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:14 smithi100 bash[31715]: cluster 2024-07-28T19:27:12.836045+0000 mgr.y (mgr.14391) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:14.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:14 smithi100 bash[31715]: audit 2024-07-28T19:27:13.963372+0000 mon.c (mon.2) 55 : audit [INF] from='client.? 172.21.15.100:0/270232258' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-28T19:27:14.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:14 smithi100 bash[31715]: audit 2024-07-28T19:27:13.964159+0000 mon.a (mon.0) 690 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-07-28T19:27:15.200 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-07-28T19:27:15.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:15 smithi138 bash[23952]: audit 2024-07-28T19:27:14.544470+0000 mon.a (mon.0) 691 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-07-28T19:27:15.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:15 smithi138 bash[23952]: cluster 2024-07-28T19:27:14.544614+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-07-28T19:27:15.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:15 smithi100 bash[21828]: audit 2024-07-28T19:27:14.544470+0000 mon.a (mon.0) 691 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-07-28T19:27:15.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:15 smithi100 bash[21828]: cluster 2024-07-28T19:27:14.544614+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-07-28T19:27:15.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:15 smithi100 bash[31715]: audit 2024-07-28T19:27:14.544470+0000 mon.a (mon.0) 691 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-07-28T19:27:15.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:15 smithi100 bash[31715]: cluster 2024-07-28T19:27:14.544614+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-07-28T19:27:16.546 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:27:16 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:27:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:27:16.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:16 smithi138 bash[23952]: cluster 2024-07-28T19:27:14.836695+0000 mgr.y (mgr.14391) 45 : cluster [DBG] pgmap v26: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:16 smithi138 bash[23952]: cluster 2024-07-28T19:27:15.538098+0000 mon.a (mon.0) 693 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:27:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:16 smithi138 bash[23952]: cluster 2024-07-28T19:27:15.545204+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-07-28T19:27:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:16 smithi138 bash[23952]: audit 2024-07-28T19:27:15.911389+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:16 smithi138 bash[23952]: audit 2024-07-28T19:27:15.913547+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[21828]: cluster 2024-07-28T19:27:14.836695+0000 mgr.y (mgr.14391) 45 : cluster [DBG] pgmap v26: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[21828]: cluster 2024-07-28T19:27:15.538098+0000 mon.a (mon.0) 693 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[21828]: cluster 2024-07-28T19:27:15.545204+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[21828]: audit 2024-07-28T19:27:15.911389+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[21828]: audit 2024-07-28T19:27:15.913547+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[31715]: cluster 2024-07-28T19:27:14.836695+0000 mgr.y (mgr.14391) 45 : cluster [DBG] pgmap v26: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[31715]: cluster 2024-07-28T19:27:15.538098+0000 mon.a (mon.0) 693 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[31715]: cluster 2024-07-28T19:27:15.545204+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-07-28T19:27:16.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[31715]: audit 2024-07-28T19:27:15.911389+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:16.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:16 smithi100 bash[31715]: audit 2024-07-28T19:27:15.913547+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:17.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:17 smithi100 bash[21828]: cluster 2024-07-28T19:27:16.565638+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-07-28T19:27:17.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:17 smithi100 bash[31715]: cluster 2024-07-28T19:27:16.565638+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-07-28T19:27:18.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:17 smithi138 bash[23952]: cluster 2024-07-28T19:27:16.565638+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-07-28T19:27:18.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:18 smithi100 bash[21828]: cluster 2024-07-28T19:27:16.837318+0000 mgr.y (mgr.14391) 46 : cluster [DBG] pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:18.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:18 smithi100 bash[31715]: cluster 2024-07-28T19:27:16.837318+0000 mgr.y (mgr.14391) 46 : cluster [DBG] pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:19.036 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:18 smithi138 bash[23952]: cluster 2024-07-28T19:27:16.837318+0000 mgr.y (mgr.14391) 46 : cluster [DBG] pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:19.975 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:20.836 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:20 smithi138 bash[23952]: cluster 2024-07-28T19:27:18.838034+0000 mgr.y (mgr.14391) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:20.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:20 smithi100 bash[21828]: cluster 2024-07-28T19:27:18.838034+0000 mgr.y (mgr.14391) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:20.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:20 smithi100 bash[31715]: cluster 2024-07-28T19:27:18.838034+0000 mgr.y (mgr.14391) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:21.776 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.777 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.777 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.777 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.778 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.778 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.778 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.778 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.779 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.779 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:21.779 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.055 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.056 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.056 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:22 smithi138 systemd[1]: Started Ceph grafana.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:27:22.057 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.057 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:21 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[21828]: cluster 2024-07-28T19:27:20.838643+0000 mgr.y (mgr.14391) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[21828]: audit 2024-07-28T19:27:22.052348+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[21828]: audit 2024-07-28T19:27:22.061061+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[21828]: audit 2024-07-28T19:27:22.068350+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[21828]: audit 2024-07-28T19:27:22.076887+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[21828]: audit 2024-07-28T19:27:22.116587+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[31715]: cluster 2024-07-28T19:27:20.838643+0000 mgr.y (mgr.14391) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[31715]: audit 2024-07-28T19:27:22.052348+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[31715]: audit 2024-07-28T19:27:22.061061+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:22.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[31715]: audit 2024-07-28T19:27:22.068350+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:22.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[31715]: audit 2024-07-28T19:27:22.076887+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:22.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:22 smithi100 bash[31715]: audit 2024-07-28T19:27:22.116587+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:23.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:22 smithi138 bash[23952]: cluster 2024-07-28T19:27:20.838643+0000 mgr.y (mgr.14391) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:23.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:22 smithi138 bash[23952]: audit 2024-07-28T19:27:22.052348+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:23.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:22 smithi138 bash[23952]: audit 2024-07-28T19:27:22.061061+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:23.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:22 smithi138 bash[23952]: audit 2024-07-28T19:27:22.068350+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:23.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:22 smithi138 bash[23952]: audit 2024-07-28T19:27:22.076887+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:23.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:22 smithi138 bash[23952]: audit 2024-07-28T19:27:22.116587+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:23.629 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379101753Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-07-28T19:27:23.629 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379295815Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-07-28T19:27:23.629 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379321768Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-07-28T19:27:23.629 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379334674Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-07-28T19:27:23.629 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379343475Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-07-28T19:27:23.629 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379352564Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-07-28T19:27:23.629 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379363069Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379370301Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379377832Z level=info msg="Path Home" path=/usr/share/grafana 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379385095Z level=info msg="Path Data" path=/var/lib/grafana 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379391662Z level=info msg="Path Logs" path=/var/log/grafana 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379397993Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379405502Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=settings t=2024-07-28T19:27:23.379417381Z level=info msg="App mode production" 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=sqlstore t=2024-07-28T19:27:23.379492893Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=sqlstore t=2024-07-28T19:27:23.379525965Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.380541558Z level=info msg="Starting DB migrations" 2024-07-28T19:27:23.630 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.380629007Z level=info msg="Executing migration" id="create migration_log table" 2024-07-28T19:27:23.631 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.430168122Z level=info msg="Executing migration" id="create user table" 2024-07-28T19:27:23.631 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.472202405Z level=info msg="Executing migration" id="add unique index user.login" 2024-07-28T19:27:23.631 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.514222003Z level=info msg="Executing migration" id="add unique index user.email" 2024-07-28T19:27:23.631 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.539525961Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-07-28T19:27:23.631 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.60654488Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-07-28T19:27:23.631 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.631784023Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-07-28T19:27:23.897 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.657041833Z level=info msg="Executing migration" id="create user table v2" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.682275235Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.724246866Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.74963513Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.774757468Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.80008088Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.825241127Z level=info msg="Executing migration" id="Update user table charset" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.850524856Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.875752872Z level=info msg="Executing migration" id="Add missing user data" 2024-07-28T19:27:23.898 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.900988405Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.92626223Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.968235972Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:23 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:23.993448301Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.02737786Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.052657816Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.077948125Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.103181053Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.145031546Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-07-28T19:27:24.167 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.170581435Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-07-28T19:27:24.455 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.212330501Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-07-28T19:27:24.455 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.237578954Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-07-28T19:27:24.455 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.262842901Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-07-28T19:27:24.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.288124224Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-07-28T19:27:24.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.31322134Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-07-28T19:27:24.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.355165589Z level=info msg="Executing migration" id="create temp_user v2" 2024-07-28T19:27:24.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.372224488Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-07-28T19:27:24.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.414378798Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-07-28T19:27:24.456 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.458719051Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-07-28T19:27:24.591 INFO:teuthology.orchestra.run.smithi100.stderr:enabled application 'rbd' on pool 'rbd' 2024-07-28T19:27:24.722 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[23952]: cluster 2024-07-28T19:27:22.839087+0000 mgr.y (mgr.14391) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:24.722 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[23952]: audit 2024-07-28T19:27:23.596315+0000 mon.a (mon.0) 701 : audit [INF] from='client.? 172.21.15.100:0/1678648508' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.481647475Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.523725746Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.565699794Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.607868466Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.624800005Z level=info msg="Executing migration" id="create star table" 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.650113427Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.675824946Z level=info msg="Executing migration" id="create org table v1" 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.700589987Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-07-28T19:27:24.723 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.725993572Z level=info msg="Executing migration" id="create org_user table v1" 2024-07-28T19:27:24.830 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:24 smithi100 bash[21828]: cluster 2024-07-28T19:27:22.839087+0000 mgr.y (mgr.14391) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:24.830 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:24 smithi100 bash[21828]: audit 2024-07-28T19:27:23.596315+0000 mon.a (mon.0) 701 : audit [INF] from='client.? 172.21.15.100:0/1678648508' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:27:24.830 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:24 smithi100 bash[31715]: cluster 2024-07-28T19:27:22.839087+0000 mgr.y (mgr.14391) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:24.830 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:24 smithi100 bash[31715]: audit 2024-07-28T19:27:23.596315+0000 mon.a (mon.0) 701 : audit [INF] from='client.? 172.21.15.100:0/1678648508' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.751238239Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.776453583Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.801737972Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.826932483Z level=info msg="Executing migration" id="Update org table charset" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.852349061Z level=info msg="Executing migration" id="Update org_user table charset" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.877512648Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.902780442Z level=info msg="Executing migration" id="create dashboard table" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.928168046Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.953327612Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-07-28T19:27:24.992 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:24 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:24.995298871Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-07-28T19:27:25.261 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.020629352Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-07-28T19:27:25.261 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.046078294Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-07-28T19:27:25.261 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.071340014Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-07-28T19:27:25.261 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.096490723Z level=info msg="Executing migration" id="create dashboard v2" 2024-07-28T19:27:25.261 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.121871149Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-07-28T19:27:25.262 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.163865175Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-07-28T19:27:25.262 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.18909069Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-07-28T19:27:25.262 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.214309334Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-07-28T19:27:25.262 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.239578762Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-07-28T19:27:25.262 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.264693638Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-07-28T19:27:25.318 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-07-28T19:27:25.318 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-07-28T19:27:25.318 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph mgr dump --format=json 2024-07-28T19:27:25.530 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.289936282Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-07-28T19:27:25.530 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.332051686Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-07-28T19:27:25.530 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.374092822Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-07-28T19:27:25.530 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.399295166Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-07-28T19:27:25.531 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.441170215Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-07-28T19:27:25.531 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.466508334Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-07-28T19:27:25.531 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.508332967Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-07-28T19:27:25.531 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.533677466Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-07-28T19:27:25.791 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[23952]: audit 2024-07-28T19:27:24.597751+0000 mon.a (mon.0) 702 : audit [INF] from='client.? 172.21.15.100:0/1678648508' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-07-28T19:27:25.791 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[23952]: cluster 2024-07-28T19:27:24.597918+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-07-28T19:27:25.791 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.558913544Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-07-28T19:27:25.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.584276979Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-07-28T19:27:25.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.626263785Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-07-28T19:27:25.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.651522586Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-07-28T19:27:25.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.676665081Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-07-28T19:27:25.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.701966339Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-07-28T19:27:25.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.744241824Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-07-28T19:27:25.792 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.769525314Z level=info msg="Executing migration" id="Update dashboard title length" 2024-07-28T19:27:25.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:25 smithi100 bash[31715]: audit 2024-07-28T19:27:24.597751+0000 mon.a (mon.0) 702 : audit [INF] from='client.? 172.21.15.100:0/1678648508' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-07-28T19:27:25.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:25 smithi100 bash[31715]: cluster 2024-07-28T19:27:24.597918+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-07-28T19:27:25.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:25 smithi100 bash[21828]: audit 2024-07-28T19:27:24.597751+0000 mon.a (mon.0) 702 : audit [INF] from='client.? 172.21.15.100:0/1678648508' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-07-28T19:27:25.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:25 smithi100 bash[21828]: cluster 2024-07-28T19:27:24.597918+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.794762065Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.820050942Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.845328805Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.862173863Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.879090369Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.929961297Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:25 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:25.963035871Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.004913046Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-07-28T19:27:26.052 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.030188124Z level=info msg="Executing migration" id="Add check_sum column" 2024-07-28T19:27:26.053 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.055427672Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-07-28T19:27:26.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:27:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:27:26.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.080535126Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-07-28T19:27:26.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.105895739Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-07-28T19:27:26.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.147679636Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-07-28T19:27:26.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.1729759Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-07-28T19:27:26.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.215243347Z level=info msg="Executing migration" id="create data_source table" 2024-07-28T19:27:26.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.240462146Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-07-28T19:27:26.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.26574857Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-07-28T19:27:26.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.307704084Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-07-28T19:27:26.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.332974776Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-07-28T19:27:26.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.374923417Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-07-28T19:27:26.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.41694974Z level=info msg="Executing migration" id="create data_source table v2" 2024-07-28T19:27:26.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.458877099Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-07-28T19:27:26.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.484165848Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-07-28T19:27:26.556 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.509457661Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-07-28T19:27:26.556 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.53468796Z level=info msg="Executing migration" id="Add column with_credentials" 2024-07-28T19:27:26.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[23952]: cluster 2024-07-28T19:27:24.839610+0000 mgr.y (mgr.14391) 50 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:26.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[23952]: cluster 2024-07-28T19:27:25.593142+0000 mon.a (mon.0) 704 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:27:26.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[23952]: cluster 2024-07-28T19:27:25.593194+0000 mon.a (mon.0) 705 : cluster [INF] Cluster is now healthy 2024-07-28T19:27:26.859 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[23952]: audit 2024-07-28T19:27:25.936916+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.576490884Z level=info msg="Executing migration" id="Add secure json data column" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.618641512Z level=info msg="Executing migration" id="Update data_source table charset" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.643899171Z level=info msg="Executing migration" id="Update initial version to 1" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.67773452Z level=info msg="Executing migration" id="Add read_only data column" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.719679631Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.744961762Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.770201054Z level=info msg="Executing migration" id="Add uid column" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.795461579Z level=info msg="Executing migration" id="Update uid value" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.820630315Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-07-28T19:27:26.860 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.862799216Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-07-28T19:27:26.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:26 smithi100 bash[21828]: cluster 2024-07-28T19:27:24.839610+0000 mgr.y (mgr.14391) 50 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:26.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:26 smithi100 bash[21828]: cluster 2024-07-28T19:27:25.593142+0000 mon.a (mon.0) 704 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:27:26.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:26 smithi100 bash[21828]: cluster 2024-07-28T19:27:25.593194+0000 mon.a (mon.0) 705 : cluster [INF] Cluster is now healthy 2024-07-28T19:27:26.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:26 smithi100 bash[21828]: audit 2024-07-28T19:27:25.936916+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:26.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:26 smithi100 bash[31715]: cluster 2024-07-28T19:27:24.839610+0000 mgr.y (mgr.14391) 50 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:26.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:26 smithi100 bash[31715]: cluster 2024-07-28T19:27:25.593142+0000 mon.a (mon.0) 704 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:27:26.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:26 smithi100 bash[31715]: cluster 2024-07-28T19:27:25.593194+0000 mon.a (mon.0) 705 : cluster [INF] Cluster is now healthy 2024-07-28T19:27:26.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:26 smithi100 bash[31715]: audit 2024-07-28T19:27:25.936916+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:27.127 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.904623722Z level=info msg="Executing migration" id="create api_key table" 2024-07-28T19:27:27.128 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.930197897Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-07-28T19:27:27.128 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.955143538Z level=info msg="Executing migration" id="add index api_key.key" 2024-07-28T19:27:27.128 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:26 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:26.997187194Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-07-28T19:27:27.128 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.014200119Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-07-28T19:27:27.128 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.081097743Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-07-28T19:27:27.128 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.106423225Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-07-28T19:27:27.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.13167266Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-07-28T19:27:27.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.156932348Z level=info msg="Executing migration" id="create api_key table v2" 2024-07-28T19:27:27.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.182133793Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-07-28T19:27:27.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.207441404Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-07-28T19:27:27.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.249408604Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-07-28T19:27:27.380 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.291606229Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-07-28T19:27:27.381 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.316890883Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-07-28T19:27:27.381 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.342108918Z level=info msg="Executing migration" id="Update api_key table charset" 2024-07-28T19:27:27.381 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.384148573Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-07-28T19:27:27.650 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.426509677Z level=info msg="Executing migration" id="Add service account foreign key" 2024-07-28T19:27:27.650 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.468507575Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-07-28T19:27:27.650 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.519188478Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-07-28T19:27:27.650 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.53609254Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-07-28T19:27:27.650 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.561550368Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-07-28T19:27:27.650 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.586705099Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-07-28T19:27:27.650 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.628808174Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-07-28T19:27:27.650 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.653934536Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-07-28T19:27:27.986 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.679242232Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-07-28T19:27:27.986 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.721268727Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-07-28T19:27:27.986 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.74661772Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-07-28T19:27:27.986 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.771730404Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-07-28T19:27:27.986 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.797071535Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-07-28T19:27:27.986 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.893744003Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-07-28T19:27:27.986 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:27 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:27.989395428Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-07-28T19:27:28.245 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.13575071Z level=info msg="Executing migration" id="create quota table v1" 2024-07-28T19:27:28.246 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.173576844Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-07-28T19:27:28.246 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.198846903Z level=info msg="Executing migration" id="Update quota table charset" 2024-07-28T19:27:28.246 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.224061403Z level=info msg="Executing migration" id="create plugin_setting table" 2024-07-28T19:27:28.246 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.249370219Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-07-28T19:27:28.515 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.274593059Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-07-28T19:27:28.515 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.316520503Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-07-28T19:27:28.515 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.341889378Z level=info msg="Executing migration" id="create session table" 2024-07-28T19:27:28.515 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.367376027Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-07-28T19:27:28.515 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.409166237Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-07-28T19:27:28.515 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.451187823Z level=info msg="Executing migration" id="create playlist table v2" 2024-07-28T19:27:28.515 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.476530221Z level=info msg="Executing migration" id="create playlist item table v2" 2024-07-28T19:27:28.515 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.518474502Z level=info msg="Executing migration" id="Update playlist table charset" 2024-07-28T19:27:28.787 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[23952]: cluster 2024-07-28T19:27:26.840249+0000 mgr.y (mgr.14391) 51 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:28.787 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[23952]: audit 2024-07-28T19:27:27.700645+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.787 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[23952]: audit 2024-07-28T19:27:27.714277+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.787 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[23952]: audit 2024-07-28T19:27:28.576214+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.787 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[23952]: audit 2024-07-28T19:27:28.585753+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.787 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[23952]: audit 2024-07-28T19:27:28.589338+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:28.787 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[23952]: audit 2024-07-28T19:27:28.590067+0000 mon.c (mon.2) 59 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:28.788 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[23952]: audit 2024-07-28T19:27:28.596837+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.543751831Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.568979925Z level=info msg="Executing migration" id="drop preferences table v2" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.613126049Z level=info msg="Executing migration" id="drop preferences table v3" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.638747301Z level=info msg="Executing migration" id="create preferences table v3" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.680622672Z level=info msg="Executing migration" id="Update preferences table charset" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.706008492Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.731300528Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.748292954Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.765238815Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-07-28T19:27:28.788 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.790516211Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-07-28T19:27:28.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[21828]: cluster 2024-07-28T19:27:26.840249+0000 mgr.y (mgr.14391) 51 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:28.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[21828]: audit 2024-07-28T19:27:27.700645+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[21828]: audit 2024-07-28T19:27:27.714277+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[21828]: audit 2024-07-28T19:27:28.576214+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[21828]: audit 2024-07-28T19:27:28.585753+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[21828]: audit 2024-07-28T19:27:28.589338+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:28.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[21828]: audit 2024-07-28T19:27:28.590067+0000 mon.c (mon.2) 59 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:28.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[21828]: audit 2024-07-28T19:27:28.596837+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[31715]: cluster 2024-07-28T19:27:26.840249+0000 mgr.y (mgr.14391) 51 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:28.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[31715]: audit 2024-07-28T19:27:27.700645+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[31715]: audit 2024-07-28T19:27:27.714277+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[31715]: audit 2024-07-28T19:27:28.576214+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[31715]: audit 2024-07-28T19:27:28.585753+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:28.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[31715]: audit 2024-07-28T19:27:28.589338+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:28.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[31715]: audit 2024-07-28T19:27:28.590067+0000 mon.c (mon.2) 59 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:28.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:28 smithi100 bash[31715]: audit 2024-07-28T19:27:28.596837+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:29.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.815796703Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-07-28T19:27:29.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.841125268Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-07-28T19:27:29.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.866520519Z level=info msg="Executing migration" id="create alert table v1" 2024-07-28T19:27:29.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.891732893Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-07-28T19:27:29.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.917061565Z level=info msg="Executing migration" id="add index alert state" 2024-07-28T19:27:29.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.95911787Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-07-28T19:27:29.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:28 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:28.984400492Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-07-28T19:27:29.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.009741031Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-07-28T19:27:29.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.035004648Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-07-28T19:27:29.343 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.077015535Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-07-28T19:27:29.343 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.1023663Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-07-28T19:27:29.343 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.127694057Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-07-28T19:27:29.343 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.152988273Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-07-28T19:27:29.344 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.178249787Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-07-28T19:27:29.344 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.220350649Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-07-28T19:27:29.344 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.245709918Z level=info msg="Executing migration" id="Add column is_default" 2024-07-28T19:27:29.344 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.27085637Z level=info msg="Executing migration" id="Add column frequency" 2024-07-28T19:27:29.344 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.304810857Z level=info msg="Executing migration" id="Add column send_reminder" 2024-07-28T19:27:29.344 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.34689503Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-07-28T19:27:29.596 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.37220122Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-07-28T19:27:29.596 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.414351095Z level=info msg="Executing migration" id="Update alert table charset" 2024-07-28T19:27:29.596 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.456276813Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-07-28T19:27:29.596 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.498383736Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-07-28T19:27:29.596 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.523762571Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-07-28T19:27:29.597 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.548994229Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-07-28T19:27:29.597 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.574287039Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-07-28T19:27:29.597 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.599663361Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-07-28T19:27:29.849 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.641652378Z level=info msg="Executing migration" id="Add for to alert table" 2024-07-28T19:27:29.849 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.66696822Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-07-28T19:27:29.849 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.68395917Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-07-28T19:27:29.849 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.700840153Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-07-28T19:27:29.849 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.717872225Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-07-28T19:27:29.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.734801989Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-07-28T19:27:29.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.776859026Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-07-28T19:27:29.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.802218885Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-07-28T19:27:29.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.827512911Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-07-28T19:27:29.850 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.852810524Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-07-28T19:27:30.077 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:30.135 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.878245755Z level=info msg="Executing migration" id="create annotation table v5" 2024-07-28T19:27:30.136 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.903450966Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-07-28T19:27:30.136 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.928624654Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-07-28T19:27:30.136 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.954005233Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-07-28T19:27:30.136 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:29 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:29.979324719Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-07-28T19:27:30.136 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.029684706Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-07-28T19:27:30.136 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.072030719Z level=info msg="Executing migration" id="Update annotation table charset" 2024-07-28T19:27:30.136 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.097259071Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-07-28T19:27:30.136 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.139258943Z level=info msg="Executing migration" id="Drop category_id index" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.164526087Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.189859464Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.215338645Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.240554253Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.265868852Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.307974247Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.350065278Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.375492469Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-07-28T19:27:30.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.417484961Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-07-28T19:27:30.683 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[23952]: cephadm 2024-07-28T19:27:28.635225+0000 mgr.y (mgr.14391) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-28T19:27:30.683 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[23952]: cephadm 2024-07-28T19:27:28.640354+0000 mgr.y (mgr.14391) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi100 2024-07-28T19:27:30.683 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[23952]: cluster 2024-07-28T19:27:28.840612+0000 mgr.y (mgr.14391) 54 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:30.683 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.459559913Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-07-28T19:27:30.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.484836933Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-07-28T19:27:30.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.526933554Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-07-28T19:27:30.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.552188865Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-07-28T19:27:30.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.577506462Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-07-28T19:27:30.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.602853871Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-07-28T19:27:30.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.645033677Z level=info msg="Executing migration" id="Add epoch_end column" 2024-07-28T19:27:30.684 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.686834236Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-07-28T19:27:30.945 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.712246108Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-07-28T19:27:30.945 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.729084474Z level=info msg="Executing migration" id="Move region to single row" 2024-07-28T19:27:30.945 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.754378565Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-07-28T19:27:30.945 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.779756143Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-07-28T19:27:30.945 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.805287753Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-07-28T19:27:30.945 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.830601704Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-07-28T19:27:30.945 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.85592706Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-07-28T19:27:30.945 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.881194011Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-07-28T19:27:30.946 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.923225009Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-07-28T19:27:30.946 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.948648038Z level=info msg="Executing migration" id="create test_data table" 2024-07-28T19:27:30.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:30 smithi100 bash[21828]: cephadm 2024-07-28T19:27:28.635225+0000 mgr.y (mgr.14391) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-28T19:27:30.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:30 smithi100 bash[21828]: cephadm 2024-07-28T19:27:28.640354+0000 mgr.y (mgr.14391) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi100 2024-07-28T19:27:30.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:30 smithi100 bash[21828]: cluster 2024-07-28T19:27:28.840612+0000 mgr.y (mgr.14391) 54 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:30.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:30 smithi100 bash[31715]: cephadm 2024-07-28T19:27:28.635225+0000 mgr.y (mgr.14391) 52 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-28T19:27:30.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:30 smithi100 bash[31715]: cephadm 2024-07-28T19:27:28.640354+0000 mgr.y (mgr.14391) 53 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi100 2024-07-28T19:27:30.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:30 smithi100 bash[31715]: cluster 2024-07-28T19:27:28.840612+0000 mgr.y (mgr.14391) 54 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:31.206 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:30 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:30.982528634Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-07-28T19:27:31.206 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.007811911Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-07-28T19:27:31.206 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.03307445Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-07-28T19:27:31.206 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.058395964Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-07-28T19:27:31.206 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.083625009Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-07-28T19:27:31.206 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.108873301Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-07-28T19:27:31.206 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.1844028Z level=info msg="Executing migration" id="create team table" 2024-07-28T19:27:31.475 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.209667896Z level=info msg="Executing migration" id="add index team.org_id" 2024-07-28T19:27:31.476 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.235043269Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-07-28T19:27:31.476 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.260174193Z level=info msg="Executing migration" id="create team member table" 2024-07-28T19:27:31.476 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.302261758Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-07-28T19:27:31.476 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.327591175Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-07-28T19:27:31.476 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.369649424Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-07-28T19:27:31.476 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.411638245Z level=info msg="Executing migration" id="Add column email to team table" 2024-07-28T19:27:31.476 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.453748816Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-07-28T19:27:31.476 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.47908511Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-07-28T19:27:31.728 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.5211008Z level=info msg="Executing migration" id="create dashboard acl table" 2024-07-28T19:27:31.728 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.546455964Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-07-28T19:27:31.728 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.597178973Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-07-28T19:27:31.728 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.639269822Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-07-28T19:27:31.729 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.664447495Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-07-28T19:27:31.729 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.689882704Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-07-28T19:27:31.729 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.73190261Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-07-28T19:27:31.990 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[23952]: cluster 2024-07-28T19:27:30.841163+0000 mgr.y (mgr.14391) 55 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:31.990 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[23952]: audit 2024-07-28T19:27:30.911011+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:31.990 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[23952]: audit 2024-07-28T19:27:30.916864+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:31.990 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.757190592Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-07-28T19:27:31.990 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.782489466Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-07-28T19:27:31.990 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.82450154Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-07-28T19:27:31.990 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.866659381Z level=info msg="Executing migration" id="create tag table" 2024-07-28T19:27:31.990 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.883686484Z level=info msg="Executing migration" id="add index tag.key_value" 2024-07-28T19:27:31.991 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.900512918Z level=info msg="Executing migration" id="create login attempt table" 2024-07-28T19:27:31.991 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.942820463Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-07-28T19:27:31.991 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.968124427Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-07-28T19:27:31.991 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:31 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:31.993394425Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-07-28T19:27:32.242 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.027026987Z level=info msg="Executing migration" id="create login_attempt v2" 2024-07-28T19:27:32.243 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.05226696Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-07-28T19:27:32.243 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.077578931Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-07-28T19:27:32.243 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.11957413Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-07-28T19:27:32.243 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.144870511Z level=info msg="Executing migration" id="create user auth table" 2024-07-28T19:27:32.243 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.170250785Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-07-28T19:27:32.243 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.195510591Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-07-28T19:27:32.243 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.220766837Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-07-28T19:27:32.243 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.246058465Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-07-28T19:27:32.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:31 smithi100 bash[31715]: cluster 2024-07-28T19:27:30.841163+0000 mgr.y (mgr.14391) 55 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:32.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:31 smithi100 bash[31715]: audit 2024-07-28T19:27:30.911011+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:32.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:31 smithi100 bash[31715]: audit 2024-07-28T19:27:30.916864+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:31 smithi100 bash[21828]: cluster 2024-07-28T19:27:30.841163+0000 mgr.y (mgr.14391) 55 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:31 smithi100 bash[21828]: audit 2024-07-28T19:27:30.911011+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:31 smithi100 bash[21828]: audit 2024-07-28T19:27:30.916864+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:32.521 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.271361308Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-07-28T19:27:32.521 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.296529211Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-07-28T19:27:32.522 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.313847578Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-07-28T19:27:32.522 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.347543848Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-07-28T19:27:32.522 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.38968951Z level=info msg="Executing migration" id="create server_lock table" 2024-07-28T19:27:32.522 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.431753341Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-07-28T19:27:32.522 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.457239089Z level=info msg="Executing migration" id="create user auth token table" 2024-07-28T19:27:32.522 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.482544885Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-07-28T19:27:32.522 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.524556749Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-07-28T19:27:32.774 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.566664195Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-07-28T19:27:32.774 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.592005566Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-07-28T19:27:32.774 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.617191177Z level=info msg="Executing migration" id="create cache_data table" 2024-07-28T19:27:32.774 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.642609163Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-07-28T19:27:32.774 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.68461816Z level=info msg="Executing migration" id="create short_url table v1" 2024-07-28T19:27:32.774 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.726805976Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-07-28T19:27:32.774 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.752107885Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-07-28T19:27:32.774 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.777324231Z level=info msg="Executing migration" id="delete alert_definition table" 2024-07-28T19:27:33.035 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.802615176Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-07-28T19:27:33.035 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.827994017Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-07-28T19:27:33.035 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.853210677Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-07-28T19:27:33.035 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.878543095Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-07-28T19:27:33.035 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.920487263Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-07-28T19:27:33.035 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.945969713Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-07-28T19:27:33.035 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:32 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:32.97130442Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-07-28T19:27:33.035 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.013314145Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-07-28T19:27:33.036 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.03868416Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-07-28T19:27:33.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.072462104Z level=info msg="Executing migration" id="drop alert_definition table" 2024-07-28T19:27:33.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.106246021Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-07-28T19:27:33.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.123223796Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-07-28T19:27:33.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.148423227Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-07-28T19:27:33.287 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.173775432Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-07-28T19:27:33.547 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.290698418Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-07-28T19:27:33.547 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.399619031Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-07-28T19:27:33.547 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.525296437Z level=info msg="Executing migration" id="create alert_instance table" 2024-07-28T19:27:33.547 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.550622696Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-07-28T19:27:33.799 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.575917235Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-07-28T19:27:33.800 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.601204755Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-07-28T19:27:33.800 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.626520337Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-07-28T19:27:33.800 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.651802863Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-07-28T19:27:33.800 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.677059833Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-07-28T19:27:33.800 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.718939617Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-07-28T19:27:33.800 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.77780672Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-07-28T19:27:33.800 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.803129729Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-07-28T19:27:34.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[23952]: cluster 2024-07-28T19:27:32.841846+0000 mgr.y (mgr.14391) 56 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:34.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.828389552Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-07-28T19:27:34.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.853750661Z level=info msg="Executing migration" id="create alert_rule table" 2024-07-28T19:27:34.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.87902353Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-07-28T19:27:34.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.904342436Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-07-28T19:27:34.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.946469416Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-07-28T19:27:34.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.971714602Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-07-28T19:27:34.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:33 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:33.997021896Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-07-28T19:27:34.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.022732188Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-07-28T19:27:34.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.047935987Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-07-28T19:27:34.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:33 smithi100 bash[21828]: cluster 2024-07-28T19:27:32.841846+0000 mgr.y (mgr.14391) 56 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:34.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:33 smithi100 bash[31715]: cluster 2024-07-28T19:27:32.841846+0000 mgr.y (mgr.14391) 56 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:34.356 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.089931844Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-07-28T19:27:34.356 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.115244667Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-07-28T19:27:34.356 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.14049626Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-07-28T19:27:34.356 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.165845373Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-07-28T19:27:34.356 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.191110933Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-07-28T19:27:34.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.216418117Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-07-28T19:27:34.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.241742873Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-07-28T19:27:34.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.267023183Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-07-28T19:27:34.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.292391046Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-07-28T19:27:34.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.317622577Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-07-28T19:27:34.357 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.359655843Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-07-28T19:27:34.625 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.376577363Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-07-28T19:27:34.626 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.410335688Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-07-28T19:27:34.626 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.435674294Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-07-28T19:27:34.626 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.477685731Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-07-28T19:27:34.626 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.519732188Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-07-28T19:27:34.626 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.561742668Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-07-28T19:27:34.626 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.587084063Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-07-28T19:27:34.626 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.629109214Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-07-28T19:27:34.879 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.65443496Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-07-28T19:27:34.879 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.679703527Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-07-28T19:27:34.879 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.705007863Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-07-28T19:27:34.879 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.730468344Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-07-28T19:27:34.879 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.75598878Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-07-28T19:27:34.879 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.781261441Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-07-28T19:27:34.879 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.80658641Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-07-28T19:27:34.879 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.831864344Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-07-28T19:27:34.880 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.857172853Z level=info msg="Executing migration" id="create provenance_type table" 2024-07-28T19:27:34.880 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.882527236Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-07-28T19:27:34.995 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:34 smithi100 systemd[1]: Stopping Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:27:34.996 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:34 smithi100 bash[56598]: ts=2024-07-28T19:27:34.935Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-07-28T19:27:35.140 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.924519713Z level=info msg="Executing migration" id="create alert_image table" 2024-07-28T19:27:35.140 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.949956344Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:34 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:34.97526255Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.000408909Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.025792822Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.051074093Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.051421434Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.093034472Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.110107737Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.127051144Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.12753641Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.128082497Z level=info msg="alerts found to migrate" alerts=0 2024-07-28T19:27:35.141 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.144095406Z level=info msg="Executing migration" id="create library_element table v1" 2024-07-28T19:27:35.299 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:27:35.418 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.161002117Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-07-28T19:27:35.418 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.17798258Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-07-28T19:27:35.419 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.194940297Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-07-28T19:27:35.419 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.220248901Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-07-28T19:27:35.419 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.29566796Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-07-28T19:27:35.419 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.337643998Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-07-28T19:27:35.419 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.379747808Z level=info msg="Executing migration" id="create data_keys table" 2024-07-28T19:27:35.419 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.421837505Z level=info msg="Executing migration" id="create secrets table" 2024-07-28T19:27:35.679 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.447179885Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-07-28T19:27:35.679 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.506141094Z level=info msg="Executing migration" id="add name column into data_keys" 2024-07-28T19:27:35.679 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.531570189Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-07-28T19:27:35.679 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.556866423Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-07-28T19:27:35.679 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.598798415Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-07-28T19:27:35.680 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.640773331Z level=info msg="Executing migration" id="create kv_store table v1" 2024-07-28T19:27:35.760 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:35 smithi100 bash[58108]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-alertmanager-a 2024-07-28T19:27:35.940 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[23952]: cluster 2024-07-28T19:27:34.842213+0000 mgr.y (mgr.14391) 57 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:35.941 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[23952]: audit 2024-07-28T19:27:35.296482+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.100:0/3645608648' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.68293707Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.708288083Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.733493868Z level=info msg="Executing migration" id="create permission table" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.758858164Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.784160105Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.809467515Z level=info msg="Executing migration" id="create role table" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.834714897Z level=info msg="Executing migration" id="add column display_name" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.859966948Z level=info msg="Executing migration" id="add column group_name" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.901981139Z level=info msg="Executing migration" id="add index role.org_id" 2024-07-28T19:27:35.941 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.944315509Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-07-28T19:27:36.211 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:27:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:27:36.211 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.96125924Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-07-28T19:27:36.211 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:35 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:35.986544407Z level=info msg="Executing migration" id="create team role table" 2024-07-28T19:27:36.211 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.020173233Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-07-28T19:27:36.211 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.045509707Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-07-28T19:27:36.211 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.087512569Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-07-28T19:27:36.211 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.112798262Z level=info msg="Executing migration" id="create user role table" 2024-07-28T19:27:36.212 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.154988215Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-07-28T19:27:36.212 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.172138885Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-07-28T19:27:36.212 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.18927162Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-07-28T19:27:36.212 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.214723128Z level=info msg="Executing migration" id="create builtin role table" 2024-07-28T19:27:36.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:35 smithi100 bash[21828]: cluster 2024-07-28T19:27:34.842213+0000 mgr.y (mgr.14391) 57 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:36.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:35 smithi100 bash[21828]: audit 2024-07-28T19:27:35.296482+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.100:0/3645608648' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:27:36.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:35 smithi100 bash[31715]: cluster 2024-07-28T19:27:34.842213+0000 mgr.y (mgr.14391) 57 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:36.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:35 smithi100 bash[31715]: audit 2024-07-28T19:27:35.296482+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.100:0/3645608648' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:27:36.244 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:35 smithi100 bash[58249]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-alertmanager-a 2024-07-28T19:27:36.244 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:36 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-07-28T19:27:36.244 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:36 smithi100 systemd[1]: Stopped Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:27:36.244 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:36 smithi100 systemd[1]: Started Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:27:36.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.239987734Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-07-28T19:27:36.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.265297429Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-07-28T19:27:36.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.30721897Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-07-28T19:27:36.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.375182209Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-07-28T19:27:36.821 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":20,"flags":0,"active_gid":14391,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6800","nonce":2345539946},{"type":"v1","addr":"172.21.15.100:6801","nonce":2345539946}]},"active_addr":"172.21.15.100:6801/2345539946","active_change":"2024-07-28T19:26:30.807599+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24280,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.100:8443/","prometheus":"http://172.21.15.100:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":3916188162}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":3784806487}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":376427751}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":480614525}]}]} 2024-07-28T19:27:36.825 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-07-28T19:27:36.825 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-07-28T19:27:36.825 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd dump --format=json 2024-07-28T19:27:36.966 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.585068951Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-07-28T19:27:36.966 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.646912982Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-07-28T19:27:37.131 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[21828]: audit 2024-07-28T19:27:36.070742+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:37.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[21828]: audit 2024-07-28T19:27:36.077160+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:37.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[21828]: cephadm 2024-07-28T19:27:36.078933+0000 mgr.y (mgr.14391) 58 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:27:37.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[21828]: cephadm 2024-07-28T19:27:36.087233+0000 mgr.y (mgr.14391) 59 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:27:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[31715]: audit 2024-07-28T19:27:36.070742+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[31715]: audit 2024-07-28T19:27:36.077160+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[31715]: cephadm 2024-07-28T19:27:36.078933+0000 mgr.y (mgr.14391) 58 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:27:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[31715]: cephadm 2024-07-28T19:27:36.087233+0000 mgr.y (mgr.14391) 59 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:27:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[23952]: audit 2024-07-28T19:27:36.070742+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[23952]: audit 2024-07-28T19:27:36.077160+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[23952]: cephadm 2024-07-28T19:27:36.078933+0000 mgr.y (mgr.14391) 58 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:27:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[23952]: cephadm 2024-07-28T19:27:36.087233+0000 mgr.y (mgr.14391) 59 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:27:37.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:36 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:36.969632111Z level=info msg="Executing migration" id="add unique index role.uid" 2024-07-28T19:27:37.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.070986506Z level=info msg="Executing migration" id="create seed assignment table" 2024-07-28T19:27:37.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.138272693Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-07-28T19:27:37.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.203712947Z level=info msg="Executing migration" id="add column hidden to role table" 2024-07-28T19:27:37.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.517926798Z level=info msg="Executing migration" id="create query_history table v1" 2024-07-28T19:27:37.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.641212049Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-07-28T19:27:37.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.70859125Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-07-28T19:27:37.994 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[58278]: ts=2024-07-28T19:27:37.801Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-07-28T19:27:37.994 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[58278]: ts=2024-07-28T19:27:37.801Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-07-28T19:27:37.994 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[58278]: ts=2024-07-28T19:27:37.803Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.100 port=9094 2024-07-28T19:27:37.994 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[58278]: ts=2024-07-28T19:27:37.804Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-07-28T19:27:37.994 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[58278]: ts=2024-07-28T19:27:37.834Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-28T19:27:37.994 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[58278]: ts=2024-07-28T19:27:37.835Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-28T19:27:37.994 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[58278]: ts=2024-07-28T19:27:37.838Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-07-28T19:27:37.994 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:37 smithi100 bash[58278]: ts=2024-07-28T19:27:37.838Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-07-28T19:27:38.073 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.808869267Z level=info msg="Executing migration" id="teams permissions migration" 2024-07-28T19:27:38.074 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.909327081Z level=info msg="Executing migration" id="dashboard permissions" 2024-07-28T19:27:38.074 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:37 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:37.959899879Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-07-28T19:27:38.074 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.043743873Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-07-28T19:27:38.326 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[23952]: cluster 2024-07-28T19:27:36.842632+0000 mgr.y (mgr.14391) 60 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:38.326 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.12373095Z level=info msg="Executing migration" id="alerting notification permissions" 2024-07-28T19:27:38.326 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.182113096Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-07-28T19:27:38.327 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.220700552Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-07-28T19:27:38.327 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.2628315Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-07-28T19:27:38.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:38 smithi100 bash[21828]: cluster 2024-07-28T19:27:36.842632+0000 mgr.y (mgr.14391) 60 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:38 smithi100 bash[31715]: cluster 2024-07-28T19:27:36.842632+0000 mgr.y (mgr.14391) 60 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:38.581 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.330311658Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-07-28T19:27:38.581 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.389787998Z level=info msg="Executing migration" id="create correlation table v1" 2024-07-28T19:27:38.581 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.484958834Z level=info msg="Executing migration" id="add index correlations.uid" 2024-07-28T19:27:38.581 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.584908235Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-07-28T19:27:39.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:38 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:38.823998185Z level=info msg="Executing migration" id="add correlation config column" 2024-07-28T19:27:39.374 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.100561841Z level=info msg="Executing migration" id="create entity_events table" 2024-07-28T19:27:39.375 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.184791163Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-07-28T19:27:39.375 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.260619545Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-07-28T19:27:39.375 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.261901141Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-07-28T19:27:39.734 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.37860256Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-07-28T19:27:39.734 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.378848269Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-07-28T19:27:39.734 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.421531637Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-07-28T19:27:39.734 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.480434217Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-07-28T19:27:40.025 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:39 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:39.737791237Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-07-28T19:27:40.076 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:39 smithi100 bash[58278]: ts=2024-07-28T19:27:39.805Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000560414s 2024-07-28T19:27:40.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[23952]: cluster 2024-07-28T19:27:38.843302+0000 mgr.y (mgr.14391) 61 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:40.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.029689947Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-07-28T19:27:40.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.159007901Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-07-28T19:27:40.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.244787554Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-07-28T19:27:40.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:40 smithi100 bash[21828]: cluster 2024-07-28T19:27:38.843302+0000 mgr.y (mgr.14391) 61 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:40.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:40 smithi100 bash[31715]: cluster 2024-07-28T19:27:38.843302+0000 mgr.y (mgr.14391) 61 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:40.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.544306106Z level=info msg="Executing migration" id="Drop public config table" 2024-07-28T19:27:40.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.653443113Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-07-28T19:27:40.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.746190961Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-07-28T19:27:41.097 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.841124406Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-07-28T19:27:41.097 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.930495176Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-07-28T19:27:41.098 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:40 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:40.972659781Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-07-28T19:27:41.098 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.052592678Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-07-28T19:27:41.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.115864732Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-07-28T19:27:41.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.157845726Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-07-28T19:27:41.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.183210145Z level=info msg="Executing migration" id="add share column" 2024-07-28T19:27:41.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.28424361Z level=info msg="Executing migration" id="create default alerting folders" 2024-07-28T19:27:41.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.326364838Z level=info msg="Executing migration" id="create file table" 2024-07-28T19:27:41.414 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.417760134Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-07-28T19:27:41.606 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:41.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.510991999Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-07-28T19:27:41.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.621936519Z level=info msg="Executing migration" id="create file_meta table" 2024-07-28T19:27:42.110 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[23952]: cluster 2024-07-28T19:27:40.843953+0000 mgr.y (mgr.14391) 62 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:42.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:41 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:41.84443217Z level=info msg="Executing migration" id="file table idx: path key" 2024-07-28T19:27:42.110 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.007468706Z level=info msg="Executing migration" id="set path collation in file table" 2024-07-28T19:27:42.389 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 systemd[1]: Stopping Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:27:42.389 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.113908202Z level=info msg="Executing migration" id="managed permissions migration" 2024-07-28T19:27:42.389 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.181575135Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-07-28T19:27:42.389 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.232782024Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-07-28T19:27:42.390 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.274771002Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-07-28T19:27:42.390 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.325876079Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-07-28T19:27:42.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:42 smithi100 bash[31715]: cluster 2024-07-28T19:27:40.843953+0000 mgr.y (mgr.14391) 62 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:42.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:42 smithi100 bash[21828]: cluster 2024-07-28T19:27:40.843953+0000 mgr.y (mgr.14391) 62 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:42.664 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.392831561Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-07-28T19:27:42.664 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.434853087Z level=info msg="Executing migration" id="update group index for alert rules" 2024-07-28T19:27:42.664 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.519109098Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-07-28T19:27:42.664 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.595378839Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-07-28T19:27:42.664 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.417Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-07-28T19:27:42.664 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.418Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-07-28T19:27:42.665 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[48170]: ts=2024-07-28T19:27:42.419Z caller=main.go:1147 level=info msg="See you next time!" 2024-07-28T19:27:43.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.667747985Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-07-28T19:27:43.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:42 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:42.854249183Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-07-28T19:27:43.451 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:43.132044791Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-07-28T19:27:43.451 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:43.239154549Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-07-28T19:27:43.451 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:43.324099847Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-07-28T19:27:43.451 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:43.374793311Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-07-28T19:27:43.451 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[50477]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-prometheus-a 2024-07-28T19:27:43.451 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[50537]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-prometheus-a 2024-07-28T19:27:43.451 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:43 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-07-28T19:27:43.451 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:43 smithi138 systemd[1]: Stopped Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:27:43.452 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:43 smithi138 systemd[1]: Started Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:27:43.687 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:43 smithi100 bash[22098]: [28/Jul/2024:19:27:43] ENGINE Bus STOPPING 2024-07-28T19:27:43.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:43.45426265Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-07-28T19:27:43.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:43.677001758Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-07-28T19:27:43.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:43 smithi100 bash[22098]: [28/Jul/2024:19:27:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:27:43.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:43 smithi100 bash[22098]: [28/Jul/2024:19:27:43] ENGINE Bus STOPPED 2024-07-28T19:27:43.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:43 smithi100 bash[22098]: [28/Jul/2024:19:27:43] ENGINE Bus STARTING 2024-07-28T19:27:43.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:43 smithi100 bash[22098]: [28/Jul/2024:19:27:43] ENGINE Serving on http://:::9283 2024-07-28T19:27:43.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:43 smithi100 bash[22098]: [28/Jul/2024:19:27:43] ENGINE Bus STARTED 2024-07-28T19:27:43.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:43 smithi100 bash[22098]: [28/Jul/2024:19:27:43] ENGINE Bus STOPPING 2024-07-28T19:27:44.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:43 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:43.953418069Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-07-28T19:27:44.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:44.071726611Z level=info msg="Executing migration" id="create folder table" 2024-07-28T19:27:44.305 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:44.138808044Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-07-28T19:27:44.689 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:44.369428111Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-07-28T19:27:44.689 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:44.507968797Z level=info msg="Executing migration" id="Update folder title length" 2024-07-28T19:27:44.689 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:44.600612355Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-07-28T19:27:44.689 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=migrator t=2024-07-28T19:27:44.692934505Z level=info msg="migrations completed" performed=485 skipped=0 duration=21.312289929s 2024-07-28T19:27:44.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: cluster 2024-07-28T19:27:42.844527+0000 mgr.y (mgr.14391) 63 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:44.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.355758+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.368364+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.377626+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:27:44.690 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.378659+0000 mgr.y (mgr.14391) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.380940+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi100.front.sepia.ceph.com:9093"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.381811+0000 mgr.y (mgr.14391) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi100.front.sepia.ceph.com:9093"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.391163+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.411295+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.412087+0000 mgr.y (mgr.14391) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.415796+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.416476+0000 mgr.y (mgr.14391) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.423161+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.442634+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.443306+0000 mgr.y (mgr.14391) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.445377+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.446347+0000 mgr.y (mgr.14391) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-07-28T19:27:44.691 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.453373+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.463655+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.773340+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.692 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[23952]: audit 2024-07-28T19:27:43.779104+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.711 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: cluster 2024-07-28T19:27:42.844527+0000 mgr.y (mgr.14391) 63 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:44.711 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.355758+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.711 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.368364+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.711 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.377626+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.378659+0000 mgr.y (mgr.14391) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.380940+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi100.front.sepia.ceph.com:9093"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.381811+0000 mgr.y (mgr.14391) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi100.front.sepia.ceph.com:9093"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.391163+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.411295+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.412087+0000 mgr.y (mgr.14391) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.415796+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.416476+0000 mgr.y (mgr.14391) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.423161+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.442634+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.443306+0000 mgr.y (mgr.14391) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.445377+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.446347+0000 mgr.y (mgr.14391) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-07-28T19:27:44.712 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.453373+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.463655+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: cluster 2024-07-28T19:27:42.844527+0000 mgr.y (mgr.14391) 63 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.355758+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.368364+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.377626+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.378659+0000 mgr.y (mgr.14391) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.380940+0000 mon.c (mon.2) 63 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi100.front.sepia.ceph.com:9093"}]: dispatch 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.381811+0000 mgr.y (mgr.14391) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi100.front.sepia.ceph.com:9093"}]: dispatch 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.391163+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.411295+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:27:44.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.412087+0000 mgr.y (mgr.14391) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.415796+0000 mon.c (mon.2) 65 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.416476+0000 mgr.y (mgr.14391) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi138.front.sepia.ceph.com:9095"}]: dispatch 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.423161+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.442634+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.443306+0000 mgr.y (mgr.14391) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.445377+0000 mon.c (mon.2) 67 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.446347+0000 mgr.y (mgr.14391) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi138.front.sepia.ceph.com:3000"}]: dispatch 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.453373+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.463655+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.773340+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.714 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[31715]: audit 2024-07-28T19:27:43.779104+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.715 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.773340+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.715 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[21828]: audit 2024-07-28T19:27:43.779104+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:44.715 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:27:44.715 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE Bus STOPPED 2024-07-28T19:27:44.715 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE Bus STARTING 2024-07-28T19:27:44.716 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE Serving on http://:::9283 2024-07-28T19:27:44.716 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE Bus STARTED 2024-07-28T19:27:44.716 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE Bus STOPPING 2024-07-28T19:27:44.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:27:44.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE Bus STOPPED 2024-07-28T19:27:44.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:44 smithi100 bash[22098]: [28/Jul/2024:19:27:44] ENGINE Bus STARTING 2024-07-28T19:27:44.999 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=sqlstore t=2024-07-28T19:27:44.694914778Z level=info msg="Created default organization" 2024-07-28T19:27:44.999 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=plugin.loader t=2024-07-28T19:27:44.887067926Z level=info msg="Plugin registered" pluginID=input 2024-07-28T19:27:44.999 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=plugin.loader t=2024-07-28T19:27:44.919647749Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-07-28T19:27:45.000 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=plugin.loader t=2024-07-28T19:27:44.919681703Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-07-28T19:27:45.000 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=secrets t=2024-07-28T19:27:44.919726125Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-07-28T19:27:45.000 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=query_data t=2024-07-28T19:27:44.921738121Z level=info msg="Query Service initialization" 2024-07-28T19:27:45.000 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:44 smithi138 bash[49893]: logger=live.push_http t=2024-07-28T19:27:44.924482454Z level=info msg="Live Push Gateway initialization" 2024-07-28T19:27:45.021 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:27:45.021 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":64,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","created":"2024-07-28T19:16:00.842363+0000","modified":"2024-07-28T19:27:24.582181+0000","last_up_change":"2024-07-28T19:25:33.970932+0000","last_in_change":"2024-07-28T19:25:08.436673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-28T19:21:19.886730+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-28T19:27:14.397490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"c4a49140-8022-42e2-bba9-333eb0e88545","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6803","nonce":2528423215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6805","nonce":2528423215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6809","nonce":2528423215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6807","nonce":2528423215}]},"public_addr":"172.21.15.100:6803/2528423215","cluster_addr":"172.21.15.100:6805/2528423215","heartbeat_back_addr":"172.21.15.100:6809/2528423215","heartbeat_front_addr":"172.21.15.100:6807/2528423215","state":["exists","up"]},{"osd":1,"uuid":"4117fbb5-c88e-4d12-bc76-77050b2c0425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6811","nonce":3874116596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6813","nonce":3874116596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6817","nonce":3874116596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6815","nonce":3874116596}]},"public_addr":"172.21.15.100:6811/3874116596","cluster_addr":"172.21.15.100:6813/3874116596","heartbeat_back_addr":"172.21.15.100:6817/3874116596","heartbeat_front_addr":"172.21.15.100:6815/3874116596","state":["exists","up"]},{"osd":2,"uuid":"f3416396-83c7-42fb-9153-9cb806281347","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6819","nonce":2139838255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6821","nonce":2139838255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6825","nonce":2139838255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6823","nonce":2139838255}]},"public_addr":"172.21.15.100:6819/2139838255","cluster_addr":"172.21.15.100:6821/2139838255","heartbeat_back_addr":"172.21.15.100:6825/2139838255","heartbeat_front_addr":"172.21.15.100:6823/2139838255","state":["exists","up"]},{"osd":3,"uuid":"0363a205-8262-42a4-aaaf-e52558fec7f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6827","nonce":684533287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6829","nonce":684533287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6832","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6833","nonce":684533287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6831","nonce":684533287}]},"public_addr":"172.21.15.100:6827/684533287","cluster_addr":"172.21.15.100:6829/684533287","heartbeat_back_addr":"172.21.15.100:6833/684533287","heartbeat_front_addr":"172.21.15.100:6831/684533287","state":["exists","up"]},{"osd":4,"uuid":"75f51136-e1cc-4d3a-89d8-1618aaff366e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6801","nonce":1306402242}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6803","nonce":1306402242}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6807","nonce":1306402242}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6805","nonce":1306402242}]},"public_addr":"172.21.15.138:6801/1306402242","cluster_addr":"172.21.15.138:6803/1306402242","heartbeat_back_addr":"172.21.15.138:6807/1306402242","heartbeat_front_addr":"172.21.15.138:6805/1306402242","state":["exists","up"]},{"osd":5,"uuid":"0a3e4950-a201-4105-9e45-ef45ce2d5f3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6809","nonce":1102784487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6811","nonce":1102784487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6815","nonce":1102784487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6813","nonce":1102784487}]},"public_addr":"172.21.15.138:6809/1102784487","cluster_addr":"172.21.15.138:6811/1102784487","heartbeat_back_addr":"172.21.15.138:6815/1102784487","heartbeat_front_addr":"172.21.15.138:6813/1102784487","state":["exists","up"]},{"osd":6,"uuid":"8a20da48-aee9-44d8-a3db-63f916c6b763","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6817","nonce":3709444988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6819","nonce":3709444988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6823","nonce":3709444988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6821","nonce":3709444988}]},"public_addr":"172.21.15.138:6817/3709444988","cluster_addr":"172.21.15.138:6819/3709444988","heartbeat_back_addr":"172.21.15.138:6823/3709444988","heartbeat_front_addr":"172.21.15.138:6821/3709444988","state":["exists","up"]},{"osd":7,"uuid":"d52b6b70-462f-42ed-966f-fa14921ba172","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6825","nonce":1158882219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6827","nonce":1158882219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6831","nonce":1158882219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6829","nonce":1158882219}]},"public_addr":"172.21.15.138:6825/1158882219","cluster_addr":"172.21.15.138:6827/1158882219","heartbeat_back_addr":"172.21.15.138:6831/1158882219","heartbeat_front_addr":"172.21.15.138:6829/1158882219","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:19:30.916069+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:20:23.352940+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:21:16.853645+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:09.540873+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:59.198788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:23:49.135324+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:24:40.795346+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:25:32.389079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.100:0/638886582":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3154304581":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6801/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/2863303498":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/989116380":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3841450082":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/553075364":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3152621533":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2291421462":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/1006451977":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3944711986":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6800/728100671":"2024-07-29T19:26:30.807202+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-28T19:27:45.305 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.003Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.003Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.003Z caller=main.go:570 level=info host_details="(Linux 5.15.0-117-generic #127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024 x86_64 smithi138 (none))" 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.003Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.003Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.005Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.005Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.007Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.007Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.009Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-07-28T19:27:45.306 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.009Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.691µs 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.009Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.010Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.011Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.011Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=42.123µs wal_replay_duration=1.698629ms wbl_replay_duration=132ns total_replay_duration=1.8579ms 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.012Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.012Z caller=main.go:1029 level=info msg="TSDB started" 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.012Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.028Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.356951ms db_storage=745ns remote_storage=1.21µs web_handler=327ns query_engine=598ns scrape=288.176µs scrape_sd=31.067µs notify=20.997µs notify_sd=6.39µs rules=15.772927ms tracing=6.217µs 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.028Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-07-28T19:27:45.307 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[50567]: ts=2024-07-28T19:27:45.028Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-07-28T19:27:45.355 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:45 smithi100 bash[22098]: [28/Jul/2024:19:27:45] ENGINE Serving on http://:::9283 2024-07-28T19:27:45.355 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:45 smithi100 bash[22098]: [28/Jul/2024:19:27:45] ENGINE Bus STARTED 2024-07-28T19:27:45.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:45 smithi100 bash[21828]: audit 2024-07-28T19:27:45.026612+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.100:0/1083114659' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:45.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:45 smithi100 bash[31715]: audit 2024-07-28T19:27:45.026612+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.100:0/1083114659' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:45.756 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-07-28T19:27:45.756 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd dump --format=json 2024-07-28T19:27:45.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:45 smithi138 bash[23952]: audit 2024-07-28T19:27:45.026612+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.100:0/1083114659' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:46.518 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[23952]: cluster 2024-07-28T19:27:44.844861+0000 mgr.y (mgr.14391) 70 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:46.518 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[23952]: audit 2024-07-28T19:27:45.904569+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:46.518 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=infra.usagestats.collector t=2024-07-28T19:27:46.162897125Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-07-28T19:27:46.518 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=provisioning.datasources t=2024-07-28T19:27:46.163643017Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-07-28T19:27:46.518 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=provisioning.datasources t=2024-07-28T19:27:46.197263018Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-07-28T19:27:46.518 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=provisioning.datasources t=2024-07-28T19:27:46.264853273Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=provisioning.alerting t=2024-07-28T19:27:46.347067043Z level=info msg="starting to provision alerting" 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=provisioning.alerting t=2024-07-28T19:27:46.347140225Z level=info msg="finished to provision alerting" 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=ngalert.state.manager t=2024-07-28T19:27:46.34741899Z level=info msg="Warming state cache for startup" 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=grafanaStorageLogger t=2024-07-28T19:27:46.348122133Z level=info msg="storage starting" 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=http.server t=2024-07-28T19:27:46.350781162Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=sqlstore.transactions t=2024-07-28T19:27:46.359126777Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=ngalert.state.manager t=2024-07-28T19:27:46.391559373Z level=info msg="State cache has been initialized" states=0 duration=44.142036ms 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=ticker t=2024-07-28T19:27:46.391639494Z level=info msg=starting first_tick=2024-07-28T19:27:50Z 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=ngalert.multiorg.alertmanager t=2024-07-28T19:27:46.391636042Z level=info msg="starting MultiOrg Alertmanager" 2024-07-28T19:27:46.519 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:27:46 smithi138 bash[49893]: logger=sqlstore.transactions t=2024-07-28T19:27:46.521676276Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-07-28T19:27:46.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:46 smithi100 bash[21828]: cluster 2024-07-28T19:27:44.844861+0000 mgr.y (mgr.14391) 70 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:46.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:46 smithi100 bash[21828]: audit 2024-07-28T19:27:45.904569+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:46.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:46 smithi100 bash[31715]: cluster 2024-07-28T19:27:44.844861+0000 mgr.y (mgr.14391) 70 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:46.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:46 smithi100 bash[31715]: audit 2024-07-28T19:27:45.904569+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:27:48.243 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:27:47 smithi100 bash[58278]: ts=2024-07-28T19:27:47.807Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002783077s 2024-07-28T19:27:48.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:48 smithi100 bash[21828]: cluster 2024-07-28T19:27:46.845437+0000 mgr.y (mgr.14391) 71 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:48.743 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:48 smithi100 bash[31715]: cluster 2024-07-28T19:27:46.845437+0000 mgr.y (mgr.14391) 71 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:48.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:48 smithi138 bash[23952]: cluster 2024-07-28T19:27:46.845437+0000 mgr.y (mgr.14391) 71 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:50.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:49 smithi138 bash[23952]: audit 2024-07-28T19:27:48.793493+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:50.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:49 smithi138 bash[23952]: audit 2024-07-28T19:27:48.801401+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:50.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:49 smithi138 bash[23952]: cluster 2024-07-28T19:27:48.845757+0000 mgr.y (mgr.14391) 72 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:50.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:49 smithi100 bash[31715]: audit 2024-07-28T19:27:48.793493+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:50.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:49 smithi100 bash[31715]: audit 2024-07-28T19:27:48.801401+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:50.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:49 smithi100 bash[31715]: cluster 2024-07-28T19:27:48.845757+0000 mgr.y (mgr.14391) 72 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:50.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:49 smithi100 bash[21828]: audit 2024-07-28T19:27:48.793493+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:50.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:49 smithi100 bash[21828]: audit 2024-07-28T19:27:48.801401+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:50.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:49 smithi100 bash[21828]: cluster 2024-07-28T19:27:48.845757+0000 mgr.y (mgr.14391) 72 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:50.323 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.327400+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.339696+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.742372+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.743760+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.753105+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.799156+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.801786+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.803170+0000 mon.c (mon.2) 74 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:51.575 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[21828]: audit 2024-07-28T19:27:50.810605+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.327400+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.339696+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.742372+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.743760+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.753105+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.799156+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.801786+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.803170+0000 mon.c (mon.2) 74 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:51.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:51 smithi100 bash[31715]: audit 2024-07-28T19:27:50.810605+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.327400+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.339696+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.742372+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.743760+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.753105+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.799156+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:27:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.801786+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:27:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.803170+0000 mon.c (mon.2) 74 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:27:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:51 smithi138 bash[23952]: audit 2024-07-28T19:27:50.810605+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:27:52.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:52 smithi100 bash[21828]: cluster 2024-07-28T19:27:50.846194+0000 mgr.y (mgr.14391) 73 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:52.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:52 smithi100 bash[31715]: cluster 2024-07-28T19:27:50.846194+0000 mgr.y (mgr.14391) 73 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:52.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:52 smithi138 bash[23952]: cluster 2024-07-28T19:27:50.846194+0000 mgr.y (mgr.14391) 73 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:53.726 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:27:53.726 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":64,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","created":"2024-07-28T19:16:00.842363+0000","modified":"2024-07-28T19:27:24.582181+0000","last_up_change":"2024-07-28T19:25:33.970932+0000","last_in_change":"2024-07-28T19:25:08.436673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-28T19:21:19.886730+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-28T19:27:14.397490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"c4a49140-8022-42e2-bba9-333eb0e88545","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6803","nonce":2528423215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6805","nonce":2528423215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6809","nonce":2528423215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6807","nonce":2528423215}]},"public_addr":"172.21.15.100:6803/2528423215","cluster_addr":"172.21.15.100:6805/2528423215","heartbeat_back_addr":"172.21.15.100:6809/2528423215","heartbeat_front_addr":"172.21.15.100:6807/2528423215","state":["exists","up"]},{"osd":1,"uuid":"4117fbb5-c88e-4d12-bc76-77050b2c0425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6811","nonce":3874116596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6813","nonce":3874116596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6817","nonce":3874116596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6815","nonce":3874116596}]},"public_addr":"172.21.15.100:6811/3874116596","cluster_addr":"172.21.15.100:6813/3874116596","heartbeat_back_addr":"172.21.15.100:6817/3874116596","heartbeat_front_addr":"172.21.15.100:6815/3874116596","state":["exists","up"]},{"osd":2,"uuid":"f3416396-83c7-42fb-9153-9cb806281347","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6819","nonce":2139838255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6821","nonce":2139838255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6825","nonce":2139838255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6823","nonce":2139838255}]},"public_addr":"172.21.15.100:6819/2139838255","cluster_addr":"172.21.15.100:6821/2139838255","heartbeat_back_addr":"172.21.15.100:6825/2139838255","heartbeat_front_addr":"172.21.15.100:6823/2139838255","state":["exists","up"]},{"osd":3,"uuid":"0363a205-8262-42a4-aaaf-e52558fec7f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6827","nonce":684533287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6829","nonce":684533287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6832","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6833","nonce":684533287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6831","nonce":684533287}]},"public_addr":"172.21.15.100:6827/684533287","cluster_addr":"172.21.15.100:6829/684533287","heartbeat_back_addr":"172.21.15.100:6833/684533287","heartbeat_front_addr":"172.21.15.100:6831/684533287","state":["exists","up"]},{"osd":4,"uuid":"75f51136-e1cc-4d3a-89d8-1618aaff366e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6801","nonce":1306402242}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6803","nonce":1306402242}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6807","nonce":1306402242}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6805","nonce":1306402242}]},"public_addr":"172.21.15.138:6801/1306402242","cluster_addr":"172.21.15.138:6803/1306402242","heartbeat_back_addr":"172.21.15.138:6807/1306402242","heartbeat_front_addr":"172.21.15.138:6805/1306402242","state":["exists","up"]},{"osd":5,"uuid":"0a3e4950-a201-4105-9e45-ef45ce2d5f3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6809","nonce":1102784487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6811","nonce":1102784487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6815","nonce":1102784487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6813","nonce":1102784487}]},"public_addr":"172.21.15.138:6809/1102784487","cluster_addr":"172.21.15.138:6811/1102784487","heartbeat_back_addr":"172.21.15.138:6815/1102784487","heartbeat_front_addr":"172.21.15.138:6813/1102784487","state":["exists","up"]},{"osd":6,"uuid":"8a20da48-aee9-44d8-a3db-63f916c6b763","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6817","nonce":3709444988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6819","nonce":3709444988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6823","nonce":3709444988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6821","nonce":3709444988}]},"public_addr":"172.21.15.138:6817/3709444988","cluster_addr":"172.21.15.138:6819/3709444988","heartbeat_back_addr":"172.21.15.138:6823/3709444988","heartbeat_front_addr":"172.21.15.138:6821/3709444988","state":["exists","up"]},{"osd":7,"uuid":"d52b6b70-462f-42ed-966f-fa14921ba172","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6825","nonce":1158882219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6827","nonce":1158882219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6831","nonce":1158882219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6829","nonce":1158882219}]},"public_addr":"172.21.15.138:6825/1158882219","cluster_addr":"172.21.15.138:6827/1158882219","heartbeat_back_addr":"172.21.15.138:6831/1158882219","heartbeat_front_addr":"172.21.15.138:6829/1158882219","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:19:30.916069+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:20:23.352940+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:21:16.853645+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:09.540873+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:59.198788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:23:49.135324+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:24:40.795346+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:25:32.389079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.100:0/638886582":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3154304581":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6801/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/2863303498":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/989116380":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3841450082":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/553075364":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3152621533":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2291421462":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/1006451977":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3944711986":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6800/728100671":"2024-07-29T19:26:30.807202+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-28T19:27:54.437 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-07-28T19:27:54.438 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-07-28T19:27:54.438 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-07-28T19:27:54.439 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-07-28T19:27:54.439 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-07-28T19:27:54.440 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-07-28T19:27:54.440 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-07-28T19:27:54.441 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-07-28T19:27:54.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:54 smithi100 bash[21828]: cluster 2024-07-28T19:27:52.846892+0000 mgr.y (mgr.14391) 74 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:54.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:54 smithi100 bash[21828]: audit 2024-07-28T19:27:53.731423+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.100:0/2881739621' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:54.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:54 smithi100 bash[31715]: cluster 2024-07-28T19:27:52.846892+0000 mgr.y (mgr.14391) 74 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:54.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:54 smithi100 bash[31715]: audit 2024-07-28T19:27:53.731423+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.100:0/2881739621' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:54.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:54 smithi138 bash[23952]: cluster 2024-07-28T19:27:52.846892+0000 mgr.y (mgr.14391) 74 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:54.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:54 smithi138 bash[23952]: audit 2024-07-28T19:27:53.731423+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.100:0/2881739621' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:27:56.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:56 smithi138 bash[23952]: cluster 2024-07-28T19:27:54.847255+0000 mgr.y (mgr.14391) 75 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:56.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:27:56 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:27:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:27:56.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:56 smithi100 bash[21828]: cluster 2024-07-28T19:27:54.847255+0000 mgr.y (mgr.14391) 75 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:56.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:56 smithi100 bash[31715]: cluster 2024-07-28T19:27:54.847255+0000 mgr.y (mgr.14391) 75 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:58.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:27:58 smithi100 bash[21828]: cluster 2024-07-28T19:27:56.847646+0000 mgr.y (mgr.14391) 76 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:58.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:27:58 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:27:58] "GET /metrics HTTP/1.1" 200 203274 "" "Prometheus/2.43.0" 2024-07-28T19:27:58.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:27:58 smithi100 bash[31715]: cluster 2024-07-28T19:27:56.847646+0000 mgr.y (mgr.14391) 76 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:58.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:27:58 smithi138 bash[23952]: cluster 2024-07-28T19:27:56.847646+0000 mgr.y (mgr.14391) 76 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:27:59.338 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:59.339 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:59.339 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:59.339 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:59.340 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:59.343 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:59.343 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:27:59.344 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:00.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:00 smithi100 bash[21828]: cluster 2024-07-28T19:27:58.848079+0000 mgr.y (mgr.14391) 77 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:00.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:00 smithi100 bash[31715]: cluster 2024-07-28T19:27:58.848079+0000 mgr.y (mgr.14391) 77 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:00.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:00 smithi138 bash[23952]: cluster 2024-07-28T19:27:58.848079+0000 mgr.y (mgr.14391) 77 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:01.668 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:01 smithi100 bash[21828]: audit 2024-07-28T19:28:00.905139+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:01.668 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:01 smithi100 bash[31715]: audit 2024-07-28T19:28:00.905139+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:01.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:01 smithi138 bash[23952]: audit 2024-07-28T19:28:00.905139+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:02.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:02 smithi100 bash[21828]: cluster 2024-07-28T19:28:00.848715+0000 mgr.y (mgr.14391) 78 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:02.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:02 smithi100 bash[31715]: cluster 2024-07-28T19:28:00.848715+0000 mgr.y (mgr.14391) 78 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:02.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:02 smithi138 bash[23952]: cluster 2024-07-28T19:28:00.848715+0000 mgr.y (mgr.14391) 78 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:04.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:03 smithi138 bash[23952]: cluster 2024-07-28T19:28:02.849408+0000 mgr.y (mgr.14391) 79 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:04.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:03 smithi100 bash[21828]: cluster 2024-07-28T19:28:02.849408+0000 mgr.y (mgr.14391) 79 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:04.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:03 smithi100 bash[31715]: cluster 2024-07-28T19:28:02.849408+0000 mgr.y (mgr.14391) 79 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:06.027 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:05 smithi100 bash[21828]: cluster 2024-07-28T19:28:04.849975+0000 mgr.y (mgr.14391) 80 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:06.027 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:05 smithi100 bash[31715]: cluster 2024-07-28T19:28:04.849975+0000 mgr.y (mgr.14391) 80 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:06.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:05 smithi138 bash[23952]: cluster 2024-07-28T19:28:04.849975+0000 mgr.y (mgr.14391) 80 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:06.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:28:06 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:28:07.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:07 smithi100 bash[21828]: cluster 2024-07-28T19:28:06.850486+0000 mgr.y (mgr.14391) 81 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:07.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:07 smithi100 bash[31715]: cluster 2024-07-28T19:28:06.850486+0000 mgr.y (mgr.14391) 81 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:08.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:07 smithi138 bash[23952]: cluster 2024-07-28T19:28:06.850486+0000 mgr.y (mgr.14391) 81 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:08.493 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:28:08 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:08] "GET /metrics HTTP/1.1" 200 203253 "" "Prometheus/2.43.0" 2024-07-28T19:28:10.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:09 smithi100 bash[21828]: cluster 2024-07-28T19:28:08.851071+0000 mgr.y (mgr.14391) 82 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:10.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:09 smithi100 bash[31715]: cluster 2024-07-28T19:28:08.851071+0000 mgr.y (mgr.14391) 82 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:10.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:09 smithi138 bash[23952]: cluster 2024-07-28T19:28:08.851071+0000 mgr.y (mgr.14391) 82 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:11.591 INFO:teuthology.orchestra.run.smithi100.stdout:55834574942 2024-07-28T19:28:11.591 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-07-28T19:28:12.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:11 smithi100 bash[21828]: cluster 2024-07-28T19:28:10.851620+0000 mgr.y (mgr.14391) 83 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:11 smithi100 bash[31715]: cluster 2024-07-28T19:28:10.851620+0000 mgr.y (mgr.14391) 83 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:12.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:11 smithi138 bash[23952]: cluster 2024-07-28T19:28:10.851620+0000 mgr.y (mgr.14391) 83 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:14.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:13 smithi100 bash[21828]: cluster 2024-07-28T19:28:12.852278+0000 mgr.y (mgr.14391) 84 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:14.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:13 smithi100 bash[31715]: cluster 2024-07-28T19:28:12.852278+0000 mgr.y (mgr.14391) 84 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:14.275 INFO:teuthology.orchestra.run.smithi100.stdout:167503724597 2024-07-28T19:28:14.276 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-07-28T19:28:14.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:13 smithi138 bash[23952]: cluster 2024-07-28T19:28:12.852278+0000 mgr.y (mgr.14391) 84 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:14.625 INFO:teuthology.orchestra.run.smithi100.stdout:236223201314 2024-07-28T19:28:14.625 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-07-28T19:28:16.021 INFO:teuthology.orchestra.run.smithi100.stdout:34359738473 2024-07-28T19:28:16.022 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-07-28T19:28:16.236 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:15 smithi100 bash[21828]: cluster 2024-07-28T19:28:14.852631+0000 mgr.y (mgr.14391) 85 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:16.236 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:15 smithi100 bash[21828]: audit 2024-07-28T19:28:15.905158+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:16.236 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:15 smithi100 bash[31715]: cluster 2024-07-28T19:28:14.852631+0000 mgr.y (mgr.14391) 85 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:16.237 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:15 smithi100 bash[31715]: audit 2024-07-28T19:28:15.905158+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:16.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:28:16 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:28:16.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:15 smithi138 bash[23952]: cluster 2024-07-28T19:28:14.852631+0000 mgr.y (mgr.14391) 85 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:16.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:15 smithi138 bash[23952]: audit 2024-07-28T19:28:15.905158+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:16.325 INFO:teuthology.orchestra.run.smithi100.stdout:133143986239 2024-07-28T19:28:16.325 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-07-28T19:28:17.464 INFO:teuthology.orchestra.run.smithi100.stdout:111669149769 2024-07-28T19:28:17.464 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-07-28T19:28:17.510 INFO:teuthology.orchestra.run.smithi100.stdout:77309411414 2024-07-28T19:28:17.511 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-07-28T19:28:17.604 INFO:teuthology.orchestra.run.smithi100.stdout:201863462956 2024-07-28T19:28:17.604 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-07-28T19:28:18.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:17 smithi100 bash[31715]: cluster 2024-07-28T19:28:16.853246+0000 mgr.y (mgr.14391) 86 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:18.175 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:17 smithi100 bash[21828]: cluster 2024-07-28T19:28:16.853246+0000 mgr.y (mgr.14391) 86 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:18.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:17 smithi138 bash[23952]: cluster 2024-07-28T19:28:16.853246+0000 mgr.y (mgr.14391) 86 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:18.493 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:28:18 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:18] "GET /metrics HTTP/1.1" 200 203250 "" "Prometheus/2.43.0" 2024-07-28T19:28:19.843 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:20.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:19 smithi100 bash[21828]: cluster 2024-07-28T19:28:18.853639+0000 mgr.y (mgr.14391) 87 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:20.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:19 smithi100 bash[31715]: cluster 2024-07-28T19:28:18.853639+0000 mgr.y (mgr.14391) 87 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:20.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:19 smithi138 bash[23952]: cluster 2024-07-28T19:28:18.853639+0000 mgr.y (mgr.14391) 87 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:20.867 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:20.868 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:20.868 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:20.868 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:21.883 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:21.884 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:21.884 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:22.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:21 smithi100 bash[21828]: cluster 2024-07-28T19:28:20.854140+0000 mgr.y (mgr.14391) 88 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:22.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:22 smithi100 bash[31715]: cluster 2024-07-28T19:28:20.854140+0000 mgr.y (mgr.14391) 88 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:22.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:21 smithi138 bash[23952]: cluster 2024-07-28T19:28:20.854140+0000 mgr.y (mgr.14391) 88 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:24.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:23 smithi100 bash[21828]: cluster 2024-07-28T19:28:22.854787+0000 mgr.y (mgr.14391) 89 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:24.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:23 smithi100 bash[31715]: cluster 2024-07-28T19:28:22.854787+0000 mgr.y (mgr.14391) 89 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:24.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:23 smithi138 bash[23952]: cluster 2024-07-28T19:28:22.854787+0000 mgr.y (mgr.14391) 89 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:26.229 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:25 smithi100 bash[21828]: cluster 2024-07-28T19:28:24.855333+0000 mgr.y (mgr.14391) 90 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:26.229 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:25 smithi100 bash[31715]: cluster 2024-07-28T19:28:24.855333+0000 mgr.y (mgr.14391) 90 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:26.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:25 smithi138 bash[23952]: cluster 2024-07-28T19:28:24.855333+0000 mgr.y (mgr.14391) 90 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:26.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:28:26 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:28:27.407 INFO:teuthology.orchestra.run.smithi100.stdout:55834574946 2024-07-28T19:28:28.174 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:28 smithi100 bash[21828]: cluster 2024-07-28T19:28:26.855984+0000 mgr.y (mgr.14391) 91 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:28.174 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:28 smithi100 bash[21828]: audit 2024-07-28T19:28:27.413915+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.100:0/2498755536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:28:28.174 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:28 smithi100 bash[31715]: cluster 2024-07-28T19:28:26.855984+0000 mgr.y (mgr.14391) 91 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:28.174 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:28 smithi100 bash[31715]: audit 2024-07-28T19:28:27.413915+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.100:0/2498755536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:28:28.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:27 smithi138 bash[23952]: cluster 2024-07-28T19:28:26.855984+0000 mgr.y (mgr.14391) 91 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:28.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:27 smithi138 bash[23952]: audit 2024-07-28T19:28:27.413915+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.100:0/2498755536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:28:28.493 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:28:28 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:28] "GET /metrics HTTP/1.1" 200 203250 "" "Prometheus/2.43.0" 2024-07-28T19:28:30.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:29 smithi100 bash[21828]: cluster 2024-07-28T19:28:28.856384+0000 mgr.y (mgr.14391) 92 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:30.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:29 smithi100 bash[31715]: cluster 2024-07-28T19:28:28.856384+0000 mgr.y (mgr.14391) 92 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:30.247 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574942 got 55834574946 for osd.1 2024-07-28T19:28:30.247 DEBUG:teuthology.parallel:result is None 2024-07-28T19:28:30.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:29 smithi138 bash[23952]: cluster 2024-07-28T19:28:28.856384+0000 mgr.y (mgr.14391) 92 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:31.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:30 smithi100 bash[21828]: audit 2024-07-28T19:28:30.905734+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:31.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:30 smithi100 bash[31715]: audit 2024-07-28T19:28:30.905734+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:31.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:30 smithi138 bash[23952]: audit 2024-07-28T19:28:30.905734+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:31 smithi100 bash[21828]: cluster 2024-07-28T19:28:30.856790+0000 mgr.y (mgr.14391) 93 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:32.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:31 smithi100 bash[31715]: cluster 2024-07-28T19:28:30.856790+0000 mgr.y (mgr.14391) 93 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:32.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:31 smithi138 bash[23952]: cluster 2024-07-28T19:28:30.856790+0000 mgr.y (mgr.14391) 93 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:33.871 INFO:teuthology.orchestra.run.smithi100.stdout:236223201318 2024-07-28T19:28:34.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:33 smithi100 bash[31715]: cluster 2024-07-28T19:28:32.857505+0000 mgr.y (mgr.14391) 94 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:34.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:33 smithi100 bash[31715]: audit 2024-07-28T19:28:33.877975+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.100:0/1388227536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:28:34.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:33 smithi100 bash[21828]: cluster 2024-07-28T19:28:32.857505+0000 mgr.y (mgr.14391) 94 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:34.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:33 smithi100 bash[21828]: audit 2024-07-28T19:28:33.877975+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.100:0/1388227536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:28:34.292 INFO:teuthology.orchestra.run.smithi100.stdout:133143986243 2024-07-28T19:28:34.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:33 smithi138 bash[23952]: cluster 2024-07-28T19:28:32.857505+0000 mgr.y (mgr.14391) 94 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:34.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:33 smithi138 bash[23952]: audit 2024-07-28T19:28:33.877975+0000 mon.c (mon.2) 80 : audit [DBG] from='client.? 172.21.15.100:0/1388227536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:28:34.562 INFO:teuthology.orchestra.run.smithi100.stdout:34359738477 2024-07-28T19:28:34.618 INFO:teuthology.orchestra.run.smithi100.stdout:167503724602 2024-07-28T19:28:35.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:34 smithi100 bash[21828]: audit 2024-07-28T19:28:34.298669+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.100:0/2469762791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:28:35.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:34 smithi100 bash[21828]: audit 2024-07-28T19:28:34.568864+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.100:0/167168014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:28:35.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:34 smithi100 bash[21828]: audit 2024-07-28T19:28:34.624869+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.100:0/829801105' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:28:35.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:34 smithi100 bash[31715]: audit 2024-07-28T19:28:34.298669+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.100:0/2469762791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:28:35.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:34 smithi100 bash[31715]: audit 2024-07-28T19:28:34.568864+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.100:0/167168014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:28:35.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:34 smithi100 bash[31715]: audit 2024-07-28T19:28:34.624869+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.100:0/829801105' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:28:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:34 smithi138 bash[23952]: audit 2024-07-28T19:28:34.298669+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.100:0/2469762791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:28:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:34 smithi138 bash[23952]: audit 2024-07-28T19:28:34.568864+0000 mon.c (mon.2) 82 : audit [DBG] from='client.? 172.21.15.100:0/167168014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:28:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:34 smithi138 bash[23952]: audit 2024-07-28T19:28:34.624869+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.100:0/829801105' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:28:35.478 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201314 got 236223201318 for osd.7 2024-07-28T19:28:35.478 DEBUG:teuthology.parallel:result is None 2024-07-28T19:28:35.903 INFO:teuthology.orchestra.run.smithi100.stdout:201863462960 2024-07-28T19:28:36.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:35 smithi100 bash[21828]: cluster 2024-07-28T19:28:34.858029+0000 mgr.y (mgr.14391) 95 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:36.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:35 smithi100 bash[21828]: audit 2024-07-28T19:28:35.910693+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.100:0/3003431278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:28:36.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:35 smithi100 bash[31715]: cluster 2024-07-28T19:28:34.858029+0000 mgr.y (mgr.14391) 95 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:36.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:35 smithi100 bash[31715]: audit 2024-07-28T19:28:35.910693+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.100:0/3003431278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:28:36.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:28:36 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:28:36.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:35 smithi138 bash[23952]: cluster 2024-07-28T19:28:34.858029+0000 mgr.y (mgr.14391) 95 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:36.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:35 smithi138 bash[23952]: audit 2024-07-28T19:28:35.910693+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.100:0/3003431278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:28:36.562 INFO:teuthology.orchestra.run.smithi100.stdout:111669149773 2024-07-28T19:28:36.573 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738473 got 34359738477 for osd.0 2024-07-28T19:28:36.573 DEBUG:teuthology.parallel:result is None 2024-07-28T19:28:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:36 smithi138 bash[23952]: audit 2024-07-28T19:28:36.568953+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.100:0/3395077740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:28:37.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:37 smithi100 bash[21828]: audit 2024-07-28T19:28:36.568953+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.100:0/3395077740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:28:37.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:37 smithi100 bash[31715]: audit 2024-07-28T19:28:36.568953+0000 mon.c (mon.2) 84 : audit [DBG] from='client.? 172.21.15.100:0/3395077740' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:28:37.940 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724597 got 167503724602 for osd.5 2024-07-28T19:28:37.940 DEBUG:teuthology.parallel:result is None 2024-07-28T19:28:37.940 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986239 got 133143986243 for osd.4 2024-07-28T19:28:37.941 DEBUG:teuthology.parallel:result is None 2024-07-28T19:28:37.997 INFO:teuthology.orchestra.run.smithi100.stdout:77309411419 2024-07-28T19:28:38.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:37 smithi100 bash[21828]: cluster 2024-07-28T19:28:36.858603+0000 mgr.y (mgr.14391) 96 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:38.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:37 smithi100 bash[31715]: cluster 2024-07-28T19:28:36.858603+0000 mgr.y (mgr.14391) 96 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:38.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:28:38 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:38] "GET /metrics HTTP/1.1" 200 203247 "" "Prometheus/2.43.0" 2024-07-28T19:28:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:37 smithi138 bash[23952]: cluster 2024-07-28T19:28:36.858603+0000 mgr.y (mgr.14391) 96 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:38.839 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149769 got 111669149773 for osd.3 2024-07-28T19:28:38.839 DEBUG:teuthology.parallel:result is None 2024-07-28T19:28:39.143 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:38 smithi100 bash[21828]: audit 2024-07-28T19:28:38.003811+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.100:0/2519731646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:28:39.404 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:38 smithi100 bash[31715]: audit 2024-07-28T19:28:38.003811+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.100:0/2519731646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:28:39.405 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462956 got 201863462960 for osd.6 2024-07-28T19:28:39.405 DEBUG:teuthology.parallel:result is None 2024-07-28T19:28:39.405 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:38 smithi138 bash[23952]: audit 2024-07-28T19:28:38.003811+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.100:0/2519731646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:28:39.547 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411414 got 77309411419 for osd.2 2024-07-28T19:28:39.547 DEBUG:teuthology.parallel:result is None 2024-07-28T19:28:39.547 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-07-28T19:28:39.548 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph pg dump --format=json 2024-07-28T19:28:40.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:40 smithi100 bash[21828]: cluster 2024-07-28T19:28:38.859112+0000 mgr.y (mgr.14391) 97 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:40.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:40 smithi100 bash[31715]: cluster 2024-07-28T19:28:38.859112+0000 mgr.y (mgr.14391) 97 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:40.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:39 smithi138 bash[23952]: cluster 2024-07-28T19:28:38.859112+0000 mgr.y (mgr.14391) 97 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:42.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:42 smithi138 bash[23952]: cluster 2024-07-28T19:28:40.859661+0000 mgr.y (mgr.14391) 98 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:42.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:42 smithi100 bash[21828]: cluster 2024-07-28T19:28:40.859661+0000 mgr.y (mgr.14391) 98 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:42.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:42 smithi100 bash[31715]: cluster 2024-07-28T19:28:40.859661+0000 mgr.y (mgr.14391) 98 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:44.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:44 smithi138 bash[23952]: cluster 2024-07-28T19:28:42.860310+0000 mgr.y (mgr.14391) 99 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:44.317 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:44.421 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:44 smithi100 bash[21828]: cluster 2024-07-28T19:28:42.860310+0000 mgr.y (mgr.14391) 99 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:44.421 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:44 smithi100 bash[31715]: cluster 2024-07-28T19:28:42.860310+0000 mgr.y (mgr.14391) 99 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:46.304 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:46 smithi100 bash[21828]: cluster 2024-07-28T19:28:44.860907+0000 mgr.y (mgr.14391) 100 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:46.304 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:46 smithi100 bash[21828]: audit 2024-07-28T19:28:45.906199+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:46.304 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:46 smithi100 bash[31715]: cluster 2024-07-28T19:28:44.860907+0000 mgr.y (mgr.14391) 100 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:46.304 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:46 smithi100 bash[31715]: audit 2024-07-28T19:28:45.906199+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:46.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:28:46 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:28:46.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:46 smithi138 bash[23952]: cluster 2024-07-28T19:28:44.860907+0000 mgr.y (mgr.14391) 100 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:46.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:46 smithi138 bash[23952]: audit 2024-07-28T19:28:45.906199+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:28:47.597 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:28:47.598 INFO:teuthology.orchestra.run.smithi100.stderr:dumped all 2024-07-28T19:28:48.165 INFO:teuthology.orchestra.run.smithi100.stdout:{"pg_ready":true,"pg_map":{"version":75,"stamp":"2024-07-28T19:28:46.861185+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385560,"kb_used_data":6712,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575656,"statfs":{"total":767960285184,"available":765517471744,"internally_reserved":0,"allocated":6873088,"data_stored":3897999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003396"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.950384+0000","last_change":"2024-07-28T19:27:15.560013+0000","last_active":"2024-07-28T19:27:16.950384+0000","last_peered":"2024-07-28T19:27:16.950384+0000","last_clean":"2024-07-28T19:27:16.950384+0000","last_became_active":"2024-07-28T19:27:15.559241+0000","last_became_peered":"2024-07-28T19:27:15.559241+0000","last_unstale":"2024-07-28T19:27:16.950384+0000","last_undegraded":"2024-07-28T19:27:16.950384+0000","last_fullsized":"2024-07-28T19:27:16.950384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:41:17.422708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:17.493098+0000","last_change":"2024-07-28T19:27:15.704938+0000","last_active":"2024-07-28T19:27:17.493098+0000","last_peered":"2024-07-28T19:27:17.493098+0000","last_clean":"2024-07-28T19:27:17.493098+0000","last_became_active":"2024-07-28T19:27:15.703726+0000","last_became_peered":"2024-07-28T19:27:15.703726+0000","last_unstale":"2024-07-28T19:27:17.493098+0000","last_undegraded":"2024-07-28T19:27:17.493098+0000","last_fullsized":"2024-07-28T19:27:17.493098+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:39:26.136768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.560262+0000","last_change":"2024-07-28T19:27:15.703022+0000","last_active":"2024-07-28T19:27:16.560262+0000","last_peered":"2024-07-28T19:27:16.560262+0000","last_clean":"2024-07-28T19:27:16.560262+0000","last_became_active":"2024-07-28T19:27:15.701965+0000","last_became_peered":"2024-07-28T19:27:15.701965+0000","last_unstale":"2024-07-28T19:27:16.560262+0000","last_undegraded":"2024-07-28T19:27:16.560262+0000","last_fullsized":"2024-07-28T19:27:16.560262+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:18:56.026871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:17.493377+0000","last_change":"2024-07-28T19:27:15.706361+0000","last_active":"2024-07-28T19:27:17.493377+0000","last_peered":"2024-07-28T19:27:17.493377+0000","last_clean":"2024-07-28T19:27:17.493377+0000","last_became_active":"2024-07-28T19:27:15.705499+0000","last_became_peered":"2024-07-28T19:27:15.705499+0000","last_unstale":"2024-07-28T19:27:17.493377+0000","last_undegraded":"2024-07-28T19:27:17.493377+0000","last_fullsized":"2024-07-28T19:27:17.493377+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:11:20.803138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:17.490634+0000","last_change":"2024-07-28T19:27:15.552985+0000","last_active":"2024-07-28T19:27:17.490634+0000","last_peered":"2024-07-28T19:27:17.490634+0000","last_clean":"2024-07-28T19:27:17.490634+0000","last_became_active":"2024-07-28T19:27:15.552104+0000","last_became_peered":"2024-07-28T19:27:15.552104+0000","last_unstale":"2024-07-28T19:27:17.490634+0000","last_undegraded":"2024-07-28T19:27:17.490634+0000","last_fullsized":"2024-07-28T19:27:17.490634+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:46:22.374425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.563143+0000","last_change":"2024-07-28T19:27:15.706055+0000","last_active":"2024-07-28T19:27:16.563143+0000","last_peered":"2024-07-28T19:27:16.563143+0000","last_clean":"2024-07-28T19:27:16.563143+0000","last_became_active":"2024-07-28T19:27:15.705102+0000","last_became_peered":"2024-07-28T19:27:15.705102+0000","last_unstale":"2024-07-28T19:27:16.563143+0000","last_undegraded":"2024-07-28T19:27:16.563143+0000","last_fullsized":"2024-07-28T19:27:16.563143+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:00:20.179809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.560117+0000","last_change":"2024-07-28T19:27:15.702756+0000","last_active":"2024-07-28T19:27:16.560117+0000","last_peered":"2024-07-28T19:27:16.560117+0000","last_clean":"2024-07-28T19:27:16.560117+0000","last_became_active":"2024-07-28T19:27:15.701931+0000","last_became_peered":"2024-07-28T19:27:15.701931+0000","last_unstale":"2024-07-28T19:27:16.560117+0000","last_undegraded":"2024-07-28T19:27:16.560117+0000","last_fullsized":"2024-07-28T19:27:16.560117+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:20:20.149727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:17.490710+0000","last_change":"2024-07-28T19:27:15.550979+0000","last_active":"2024-07-28T19:27:17.490710+0000","last_peered":"2024-07-28T19:27:17.490710+0000","last_clean":"2024-07-28T19:27:17.490710+0000","last_became_active":"2024-07-28T19:27:15.550214+0000","last_became_peered":"2024-07-28T19:27:15.550214+0000","last_unstale":"2024-07-28T19:27:17.490710+0000","last_undegraded":"2024-07-28T19:27:17.490710+0000","last_fullsized":"2024-07-28T19:27:17.490710+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:17:06.298582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.560201+0000","last_change":"2024-07-28T19:25:38.028024+0000","last_active":"2024-07-28T19:27:16.560201+0000","last_peered":"2024-07-28T19:27:16.560201+0000","last_clean":"2024-07-28T19:27:16.560201+0000","last_became_active":"2024-07-28T19:25:38.027110+0000","last_became_peered":"2024-07-28T19:25:38.027110+0000","last_unstale":"2024-07-28T19:27:16.560201+0000","last_undegraded":"2024-07-28T19:27:16.560201+0000","last_fullsized":"2024-07-28T19:27:16.560201+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_clean_scrub_stamp":"2024-07-28T19:21:20.215133+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:20:35.538874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201321,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.85899999999999999,"15min":0.871},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2030000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.627},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.91800000000000004,"15min":0.92200000000000004},"min":{"1min":0.66600000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1910000000000001,"5min":1.254,"15min":1.254},"last":1.016}]},{"osd":1,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.84699999999999998,"15min":0.85299999999999998},"min":{"1min":0.52500000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.877,"5min":0.92300000000000004,"15min":0.92400000000000004},"min":{"1min":0.56399999999999995,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2030000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.95199999999999996}]},{"osd":2,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.98599999999999999,"15min":0.98199999999999998},"min":{"1min":0.58499999999999996,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.218,"5min":1.284,"15min":1.284},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.98399999999999999,"15min":1.0049999999999999},"min":{"1min":0.56799999999999995,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.232,"5min":1.296,"15min":1.296},"last":0.76000000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.92500000000000004,"15min":0.92300000000000004},"min":{"1min":0.57199999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2509999999999999,"5min":1.262,"15min":1.262},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.95499999999999996,"15min":0.96599999999999997},"min":{"1min":0.64600000000000002,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.68200000000000005}]},{"osd":4,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.872,"15min":0.877},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":1.056},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.98099999999999998,"15min":1.008},"min":{"1min":0.40000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":1.2150000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":0.98999999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.94199999999999995,"15min":0.96399999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0309999999999999,"15min":1.04},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.256,"5min":1.262,"15min":1.262},"last":1.093}]},{"osd":6,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0700000000000001,"15min":1.0860000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.2350000000000001,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":1.3819999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0649999999999999,"15min":1.095},"min":{"1min":0.53800000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.2769999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.0389999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462963,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.81299999999999994,"15min":0.82899999999999996},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.087,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.89100000000000001,"15min":0.88900000000000001},"min":{"1min":0.53000000000000003,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.252,"5min":1.282,"15min":1.282},"last":0.86799999999999999}]},{"osd":1,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.84899999999999998,"15min":0.84699999999999998},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":1.224,"5min":1.48,"15min":1.48},"last":1.2},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.85999999999999999,"15min":0.84999999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.2789999999999999,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.81100000000000005}]},{"osd":2,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.87,"15min":0.85399999999999998},"min":{"1min":0.52500000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.1419999999999999,"5min":1.222,"15min":1.222},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.88,"5min":0.90100000000000002,"15min":0.89200000000000002},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.96199999999999997}]},{"osd":3,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.93000000000000005,"15min":0.92200000000000004},"min":{"1min":0.57899999999999996,"5min":0.498,"15min":0.498},"max":{"1min":1.3460000000000001,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.111},{"interface":"front","average":{"1min":0.872,"5min":0.86899999999999999,"15min":0.85499999999999998},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.1919999999999999,"5min":1.298,"15min":1.298},"last":1.002}]},{"osd":4,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.81299999999999994,"15min":0.80400000000000005},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.85899999999999999,"15min":0.81399999999999995},"min":{"1min":0.37,"5min":0.316,"15min":0.316},"max":{"1min":1.5589999999999999,"5min":1.617,"15min":1.617},"last":1.274}]},{"osd":5,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.85799999999999998,"15min":0.81399999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.4370000000000001,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.86899999999999999,"15min":0.81899999999999995},"min":{"1min":0.42099999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.4179999999999999,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":1.4079999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:28:36 2024","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.036,"15min":1.0509999999999999},"min":{"1min":0.44400000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.26,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.028},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0820000000000001,"15min":1.0900000000000001},"min":{"1min":0.46300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.4399999999999999,"5min":1.476,"15min":1.476},"last":1.3420000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724605,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.88600000000000001,"15min":0.85999999999999999},"min":{"1min":0.57199999999999995,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.258,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":1.083},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.84299999999999997,"15min":0.83799999999999997},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.2,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.95099999999999996}]},{"osd":1,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.88900000000000001,"15min":0.86399999999999999},"min":{"1min":0.61599999999999999,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3089999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.89400000000000002,"15min":0.86599999999999999},"min":{"1min":0.64700000000000002,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.347,"5min":1.347,"15min":1.347},"last":1.1180000000000001}]},{"osd":2,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.871,"15min":0.86199999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.274,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.1519999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.90300000000000002,"15min":0.88700000000000001},"min":{"1min":0.67800000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.2170000000000001,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.98899999999999999}]},{"osd":3,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.91300000000000003,"15min":0.89100000000000001},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.397,"5min":1.397,"15min":1.397},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.92400000000000004,"15min":0.88200000000000001},"min":{"1min":0.621,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2849999999999999,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":1.0149999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.77500000000000002,"15min":0.72299999999999998},"min":{"1min":0.42099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.81000000000000005,"15min":0.78100000000000003},"min":{"1min":0.434,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.4870000000000001,"5min":1.4870000000000001,"15min":1.4870000000000001},"last":0.78500000000000003}]},{"osd":6,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.89600000000000002,"15min":0.85199999999999998},"min":{"1min":0.60699999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.5189999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.038,"5min":0.93000000000000005,"15min":0.90700000000000003},"min":{"1min":0.66400000000000003,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.468,"5min":2.1669999999999998,"15min":2.1669999999999998},"last":0.82799999999999996}]},{"osd":7,"last update":"Sun Jul 28 19:28:41 2024","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.008,"15min":1},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.306,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.028,"15min":1.038},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.254,"5min":1.516,"15min":1.516},"last":0.96499999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986246,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.86899999999999999,"15min":0.84299999999999997},"min":{"1min":0.56200000000000006,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.2529999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.83399999999999996,"15min":0.82199999999999995},"min":{"1min":0.57899999999999996,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.1299999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":1.036}]},{"osd":1,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.82399999999999995,"15min":0.79200000000000004},"min":{"1min":0.51300000000000001,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.1000000000000001,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.88,"5min":0.88300000000000001,"15min":0.86299999999999999},"min":{"1min":0.57399999999999995,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2430000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.80100000000000005}]},{"osd":2,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.91800000000000004,"15min":0.88900000000000001},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.877,"15min":0.86199999999999999},"min":{"1min":0.61899999999999999,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.97999999999999998}]},{"osd":3,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.89600000000000002,"15min":0.86799999999999999},"min":{"1min":0.52400000000000002,"5min":0.504,"15min":0.504},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.88600000000000001,"15min":0.86299999999999999},"min":{"1min":0.64400000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.173,"5min":1.583,"15min":1.583},"last":0.93200000000000005}]},{"osd":5,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.84199999999999997,"15min":0.79600000000000004},"min":{"1min":0.42899999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.5569999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":1.0609999999999999},{"interface":"front","average":{"1min":1.008,"5min":0.85899999999999999,"15min":0.81699999999999995},"min":{"1min":0.50600000000000001,"5min":0.309,"15min":0.309},"max":{"1min":1.6919999999999999,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":1.0089999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":0.91600000000000004,"15min":0.90600000000000003},"min":{"1min":0.45700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.6519999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.879},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.88200000000000001,"15min":0.84499999999999997},"min":{"1min":0.46899999999999997,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.3440000000000001,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.1000000000000001}]},{"osd":7,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.284,"15min":1.4139999999999999},"min":{"1min":0.47999999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.22,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.4099999999999999,"15min":1.5609999999999999},"min":{"1min":0.52300000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.202,"5min":10.920999999999999,"15min":10.920999999999999},"last":1.143}]}]},{"osd":3,"up_from":26,"seq":111669149776,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.73299999999999998,"15min":0.69299999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.999,"5min":1.909,"15min":1.909},"last":0.75},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.81599999999999995,"15min":0.77700000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.3839999999999999,"5min":1.556,"15min":1.556},"last":0.79700000000000004}]},{"osd":1,"last update":"Sun Jul 28 19:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.73599999999999999,"15min":0.71199999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.476,"5min":1.583,"15min":1.583},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.71099999999999997,"15min":0.67300000000000004},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":0.59499999999999997}]},{"osd":2,"last update":"Sun Jul 28 19:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.79900000000000004,"15min":0.75900000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.839,"5min":1.839,"15min":1.839},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.751,"5min":0.74399999999999999,"15min":0.67700000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.3320000000000001,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.64200000000000002}]},{"osd":4,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.89900000000000002,"15min":0.875},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.1419999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.024,"5min":1.0009999999999999,"15min":1.075},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.996,"5min":5.6539999999999999,"15min":5.6539999999999999},"last":0.91900000000000004}]},{"osd":5,"last update":"Sun Jul 28 19:28:00 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.93000000000000005,"15min":0.92000000000000004},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.208,"5min":1.341,"15min":1.341},"last":1.01},{"interface":"front","average":{"1min":1.01,"5min":0.97799999999999998,"15min":0.96199999999999997},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2350000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.85999999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:27:50 2024","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.022,"15min":1.026},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.202,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.103},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0149999999999999,"15min":1.008},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.341,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.89700000000000002}]},{"osd":7,"last update":"Sun Jul 28 19:28:39 2024","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0760000000000001,"15min":1.089},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":2.0880000000000001,"5min":2.0880000000000001,"15min":2.0880000000000001},"last":1.046},{"interface":"front","average":{"1min":0.94099999999999995,"5min":1.0629999999999999,"15min":1.0780000000000001},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.28,"5min":1.401,"15min":1.401},"last":0.95099999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411421,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:30 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.63700000000000001,"15min":0.56699999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.82299999999999995,"5min":2.4569999999999999,"15min":2.4569999999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.79900000000000004,"15min":0.68400000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.317},"max":{"1min":1.1299999999999999,"5min":1.456,"15min":1.456},"last":0.41599999999999998}]},{"osd":1,"last update":"Sun Jul 28 19:28:30 2024","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.70199999999999996,"15min":0.59299999999999997},"min":{"1min":0.51800000000000002,"5min":0.39800000000000002,"15min":0.30599999999999999},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.502},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.76600000000000001,"15min":0.65900000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":1.2230000000000001,"5min":1.7350000000000001,"15min":1.7350000000000001},"last":0.437}]},{"osd":3,"last update":"Sun Jul 28 19:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.84499999999999997,"15min":0.75900000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.3759999999999999,"5min":1.9790000000000001,"15min":1.9790000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.84799999999999998,"15min":0.80200000000000005},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.242,"5min":1.605,"15min":1.605},"last":0.38100000000000001}]},{"osd":4,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.88200000000000001,"15min":0.85299999999999998},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.085,"5min":1.196,"15min":1.196},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.96699999999999997,"15min":0.92300000000000004},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.747}]},{"osd":5,"last update":"Sun Jul 28 19:28:02 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.98599999999999999,"15min":1.036},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.226,"5min":3.9969999999999999,"15min":3.9969999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.0529999999999999,"15min":1.1339999999999999},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.504,"5min":5.3680000000000003,"15min":5.3680000000000003},"last":0.79400000000000004}]},{"osd":6,"last update":"Sun Jul 28 19:27:49 2024","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.022,"15min":1.03},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.405,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.85299999999999998},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.0600000000000001,"15min":1.052},"min":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.6859999999999999,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.65400000000000003}]},{"osd":7,"last update":"Sun Jul 28 19:28:40 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.091,"15min":1.0980000000000001},"min":{"1min":0.751,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.319,"5min":1.766,"15min":1.766},"last":0.621},{"interface":"front","average":{"1min":1.02,"5min":1.0189999999999999,"15min":1.034},"min":{"1min":0.58699999999999997,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.331,"5min":1.454,"15min":1.454},"last":0.69399999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574950,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:42 2024","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.65400000000000003,"15min":0.55800000000000005},"min":{"1min":0.34100000000000003,"5min":0.33100000000000002,"15min":0.28399999999999997},"max":{"1min":2.2530000000000001,"5min":2.2530000000000001,"15min":2.2530000000000001},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.72399999999999998,"15min":0.60399999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.1359999999999999,"5min":1.351,"15min":1.351},"last":0.996}]},{"osd":2,"last update":"Sun Jul 28 19:28:29 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.68300000000000005,"15min":0.59799999999999998},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.77700000000000002,"15min":0.69499999999999995},"min":{"1min":0.34100000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.76500000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:28:22 2024","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.83899999999999997,"15min":0.78500000000000003},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.6659999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.76200000000000001,"15min":0.748},"min":{"1min":0.32900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.92400000000000004,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":1.1459999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:28:10 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.97299999999999998,"15min":0.91700000000000004},"min":{"1min":0.69999999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.2789999999999999,"5min":1.538,"15min":1.538},"last":1.554},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.91700000000000004,"15min":0.90200000000000002},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.53,"5min":1.53,"15min":1.53},"last":1.228}]},{"osd":5,"last update":"Sun Jul 28 19:28:02 2024","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.92300000000000004,"15min":0.90700000000000003},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.605,"5min":1.605,"15min":1.605},"last":1.4330000000000001},{"interface":"front","average":{"1min":1.018,"5min":1.0049999999999999,"15min":0.99099999999999999},"min":{"1min":0.75700000000000001,"5min":0.625,"15min":0.625},"max":{"1min":1.266,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.383}]},{"osd":6,"last update":"Sun Jul 28 19:27:47 2024","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.0920000000000001,"15min":1.0900000000000001},"min":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"max":{"1min":1.3440000000000001,"5min":1.5780000000000001,"15min":1.5780000000000001},"last":1.74},{"interface":"front","average":{"1min":1.036,"5min":0.996,"15min":0.97599999999999998},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":1.778}]},{"osd":7,"last update":"Sun Jul 28 19:28:39 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.006,"15min":1.006},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.677,"5min":1.677,"15min":1.677},"last":1.619},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0649999999999999,"15min":1.089},"min":{"1min":0.60599999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":1.304}]}]},{"osd":0,"up_from":8,"seq":34359738480,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Jul 28 19:28:44 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.64800000000000002,"15min":0.55400000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.95699999999999996,"5min":1.0569999999999999,"15min":1.127},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.74099999999999999,"15min":0.61699999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.2869999999999999,"5min":1.347,"15min":1.347},"last":0.95099999999999996}]},{"osd":2,"last update":"Sun Jul 28 19:28:30 2024","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.77100000000000002,"15min":0.66800000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.32900000000000001},"max":{"1min":1.026,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.73499999999999999,"15min":0.67200000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29499999999999998},"max":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.79600000000000004}]},{"osd":3,"last update":"Sun Jul 28 19:28:30 2024","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.755,"15min":0.73399999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.002,"5min":1.28,"15min":1.28},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.1240000000000001,"5min":1.173,"15min":1.3220000000000001},"last":0.86899999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:28:08 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.90300000000000002,"15min":0.84299999999999997},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.03},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.93100000000000005,"15min":0.89300000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.292,"5min":1.327,"15min":1.327},"last":0.83499999999999996}]},{"osd":5,"last update":"Sun Jul 28 19:27:58 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.94999999999999996,"15min":0.94199999999999995},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.94399999999999995,"15min":0.92200000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3180000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":1.175}]},{"osd":6,"last update":"Sun Jul 28 19:27:55 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.022,"15min":1.028},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.99199999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.052,"15min":1.054},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.512,"5min":1.512,"15min":1.512},"last":1.0629999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:28:44 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.006,"15min":1.012},"min":{"1min":0.58999999999999997,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3129999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":1.0920000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0289999999999999,"15min":1.0429999999999999},"min":{"1min":0.61899999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":1.1279999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-28T19:28:48.167 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph pg dump --format=json 2024-07-28T19:28:48.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:48 smithi138 bash[23952]: cluster 2024-07-28T19:28:46.861521+0000 mgr.y (mgr.14391) 101 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:48.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:48 smithi138 bash[23952]: audit 2024-07-28T19:28:47.596403+0000 mgr.y (mgr.14391) 102 : audit [DBG] from='client.14571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:28:48.493 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:28:48 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:48] "GET /metrics HTTP/1.1" 200 203263 "" "Prometheus/2.43.0" 2024-07-28T19:28:48.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:48 smithi100 bash[21828]: cluster 2024-07-28T19:28:46.861521+0000 mgr.y (mgr.14391) 101 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:48.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:48 smithi100 bash[21828]: audit 2024-07-28T19:28:47.596403+0000 mgr.y (mgr.14391) 102 : audit [DBG] from='client.14571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:28:48.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:48 smithi100 bash[31715]: cluster 2024-07-28T19:28:46.861521+0000 mgr.y (mgr.14391) 101 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:48.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:48 smithi100 bash[31715]: audit 2024-07-28T19:28:47.596403+0000 mgr.y (mgr.14391) 102 : audit [DBG] from='client.14571 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:28:50.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:50 smithi138 bash[23952]: cluster 2024-07-28T19:28:48.862070+0000 mgr.y (mgr.14391) 103 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:50.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:50 smithi100 bash[21828]: cluster 2024-07-28T19:28:48.862070+0000 mgr.y (mgr.14391) 103 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:50.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:50 smithi100 bash[31715]: cluster 2024-07-28T19:28:48.862070+0000 mgr.y (mgr.14391) 103 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:51 smithi138 bash[23952]: audit 2024-07-28T19:28:50.858485+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:28:51.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:51 smithi100 bash[21828]: audit 2024-07-28T19:28:50.858485+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:28:51.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:51 smithi100 bash[31715]: audit 2024-07-28T19:28:50.858485+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:28:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:52 smithi138 bash[23952]: cluster 2024-07-28T19:28:50.862545+0000 mgr.y (mgr.14391) 104 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:52 smithi138 bash[23952]: audit 2024-07-28T19:28:51.325703+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:28:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:52 smithi138 bash[23952]: audit 2024-07-28T19:28:51.326976+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:28:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:52 smithi138 bash[23952]: audit 2024-07-28T19:28:51.335668+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:28:52.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:52 smithi100 bash[21828]: cluster 2024-07-28T19:28:50.862545+0000 mgr.y (mgr.14391) 104 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:52 smithi100 bash[21828]: audit 2024-07-28T19:28:51.325703+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:28:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:52 smithi100 bash[21828]: audit 2024-07-28T19:28:51.326976+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:28:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:52 smithi100 bash[21828]: audit 2024-07-28T19:28:51.335668+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:28:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:52 smithi100 bash[31715]: cluster 2024-07-28T19:28:50.862545+0000 mgr.y (mgr.14391) 104 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:52 smithi100 bash[31715]: audit 2024-07-28T19:28:51.325703+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:28:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:52 smithi100 bash[31715]: audit 2024-07-28T19:28:51.326976+0000 mon.c (mon.2) 88 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:28:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:52 smithi100 bash[31715]: audit 2024-07-28T19:28:51.335668+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:28:52.933 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:28:54.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:54 smithi138 bash[23952]: cluster 2024-07-28T19:28:52.863200+0000 mgr.y (mgr.14391) 105 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:54.372 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:54 smithi100 bash[31715]: cluster 2024-07-28T19:28:52.863200+0000 mgr.y (mgr.14391) 105 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:54.372 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:54 smithi100 bash[21828]: cluster 2024-07-28T19:28:52.863200+0000 mgr.y (mgr.14391) 105 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:56.140 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:28:56.141 INFO:teuthology.orchestra.run.smithi100.stderr:dumped all 2024-07-28T19:28:56.151 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:56 smithi100 bash[21828]: cluster 2024-07-28T19:28:54.863639+0000 mgr.y (mgr.14391) 106 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:56.151 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:56 smithi100 bash[31715]: cluster 2024-07-28T19:28:54.863639+0000 mgr.y (mgr.14391) 106 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:56.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:28:56 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:28:56.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:56 smithi138 bash[23952]: cluster 2024-07-28T19:28:54.863639+0000 mgr.y (mgr.14391) 106 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:56.798 INFO:teuthology.orchestra.run.smithi100.stdout:{"pg_ready":true,"pg_map":{"version":79,"stamp":"2024-07-28T19:28:54.863446+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385560,"kb_used_data":6712,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575656,"statfs":{"total":767960285184,"available":765517471744,"internally_reserved":0,"allocated":6873088,"data_stored":3897999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003484"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.950384+0000","last_change":"2024-07-28T19:27:15.560013+0000","last_active":"2024-07-28T19:27:16.950384+0000","last_peered":"2024-07-28T19:27:16.950384+0000","last_clean":"2024-07-28T19:27:16.950384+0000","last_became_active":"2024-07-28T19:27:15.559241+0000","last_became_peered":"2024-07-28T19:27:15.559241+0000","last_unstale":"2024-07-28T19:27:16.950384+0000","last_undegraded":"2024-07-28T19:27:16.950384+0000","last_fullsized":"2024-07-28T19:27:16.950384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:41:17.422708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:17.493098+0000","last_change":"2024-07-28T19:27:15.704938+0000","last_active":"2024-07-28T19:27:17.493098+0000","last_peered":"2024-07-28T19:27:17.493098+0000","last_clean":"2024-07-28T19:27:17.493098+0000","last_became_active":"2024-07-28T19:27:15.703726+0000","last_became_peered":"2024-07-28T19:27:15.703726+0000","last_unstale":"2024-07-28T19:27:17.493098+0000","last_undegraded":"2024-07-28T19:27:17.493098+0000","last_fullsized":"2024-07-28T19:27:17.493098+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:39:26.136768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.560262+0000","last_change":"2024-07-28T19:27:15.703022+0000","last_active":"2024-07-28T19:27:16.560262+0000","last_peered":"2024-07-28T19:27:16.560262+0000","last_clean":"2024-07-28T19:27:16.560262+0000","last_became_active":"2024-07-28T19:27:15.701965+0000","last_became_peered":"2024-07-28T19:27:15.701965+0000","last_unstale":"2024-07-28T19:27:16.560262+0000","last_undegraded":"2024-07-28T19:27:16.560262+0000","last_fullsized":"2024-07-28T19:27:16.560262+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:18:56.026871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:17.493377+0000","last_change":"2024-07-28T19:27:15.706361+0000","last_active":"2024-07-28T19:27:17.493377+0000","last_peered":"2024-07-28T19:27:17.493377+0000","last_clean":"2024-07-28T19:27:17.493377+0000","last_became_active":"2024-07-28T19:27:15.705499+0000","last_became_peered":"2024-07-28T19:27:15.705499+0000","last_unstale":"2024-07-28T19:27:17.493377+0000","last_undegraded":"2024-07-28T19:27:17.493377+0000","last_fullsized":"2024-07-28T19:27:17.493377+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:11:20.803138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:17.490634+0000","last_change":"2024-07-28T19:27:15.552985+0000","last_active":"2024-07-28T19:27:17.490634+0000","last_peered":"2024-07-28T19:27:17.490634+0000","last_clean":"2024-07-28T19:27:17.490634+0000","last_became_active":"2024-07-28T19:27:15.552104+0000","last_became_peered":"2024-07-28T19:27:15.552104+0000","last_unstale":"2024-07-28T19:27:17.490634+0000","last_undegraded":"2024-07-28T19:27:17.490634+0000","last_fullsized":"2024-07-28T19:27:17.490634+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:46:22.374425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.563143+0000","last_change":"2024-07-28T19:27:15.706055+0000","last_active":"2024-07-28T19:27:16.563143+0000","last_peered":"2024-07-28T19:27:16.563143+0000","last_clean":"2024-07-28T19:27:16.563143+0000","last_became_active":"2024-07-28T19:27:15.705102+0000","last_became_peered":"2024-07-28T19:27:15.705102+0000","last_unstale":"2024-07-28T19:27:16.563143+0000","last_undegraded":"2024-07-28T19:27:16.563143+0000","last_fullsized":"2024-07-28T19:27:16.563143+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:00:20.179809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.560117+0000","last_change":"2024-07-28T19:27:15.702756+0000","last_active":"2024-07-28T19:27:16.560117+0000","last_peered":"2024-07-28T19:27:16.560117+0000","last_clean":"2024-07-28T19:27:16.560117+0000","last_became_active":"2024-07-28T19:27:15.701931+0000","last_became_peered":"2024-07-28T19:27:15.701931+0000","last_unstale":"2024-07-28T19:27:16.560117+0000","last_undegraded":"2024-07-28T19:27:16.560117+0000","last_fullsized":"2024-07-28T19:27:16.560117+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:20:20.149727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:17.490710+0000","last_change":"2024-07-28T19:27:15.550979+0000","last_active":"2024-07-28T19:27:17.490710+0000","last_peered":"2024-07-28T19:27:17.490710+0000","last_clean":"2024-07-28T19:27:17.490710+0000","last_became_active":"2024-07-28T19:27:15.550214+0000","last_became_peered":"2024-07-28T19:27:15.550214+0000","last_unstale":"2024-07-28T19:27:17.490710+0000","last_undegraded":"2024-07-28T19:27:17.490710+0000","last_fullsized":"2024-07-28T19:27:17.490710+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:17:06.298582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-07-28T19:27:16.560201+0000","last_change":"2024-07-28T19:25:38.028024+0000","last_active":"2024-07-28T19:27:16.560201+0000","last_peered":"2024-07-28T19:27:16.560201+0000","last_clean":"2024-07-28T19:27:16.560201+0000","last_became_active":"2024-07-28T19:25:38.027110+0000","last_became_peered":"2024-07-28T19:25:38.027110+0000","last_unstale":"2024-07-28T19:27:16.560201+0000","last_undegraded":"2024-07-28T19:27:16.560201+0000","last_fullsized":"2024-07-28T19:27:16.560201+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_clean_scrub_stamp":"2024-07-28T19:21:20.215133+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:20:35.538874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201322,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.85899999999999999,"15min":0.871},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2030000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.91800000000000004,"15min":0.92200000000000004},"min":{"1min":0.66600000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1910000000000001,"5min":1.254,"15min":1.254},"last":0.749}]},{"osd":1,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.84699999999999998,"15min":0.85299999999999998},"min":{"1min":0.52500000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2050000000000001,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.877,"5min":0.92300000000000004,"15min":0.92400000000000004},"min":{"1min":0.56399999999999995,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2030000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.58599999999999997}]},{"osd":2,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.98599999999999999,"15min":0.98199999999999998},"min":{"1min":0.58499999999999996,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.218,"5min":1.284,"15min":1.284},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.98399999999999999,"15min":1.0049999999999999},"min":{"1min":0.56799999999999995,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.232,"5min":1.296,"15min":1.296},"last":0.68300000000000005}]},{"osd":3,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.92500000000000004,"15min":0.92300000000000004},"min":{"1min":0.57199999999999995,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2509999999999999,"5min":1.262,"15min":1.262},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.95499999999999996,"15min":0.96599999999999997},"min":{"1min":0.64600000000000002,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.84999999999999998}]},{"osd":4,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.872,"15min":0.877},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.98099999999999998,"15min":1.008},"min":{"1min":0.40000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":1.2150000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.046}]},{"osd":5,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.94199999999999995,"15min":0.96399999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":1.01},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0309999999999999,"15min":1.04},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.256,"5min":1.262,"15min":1.262},"last":1.0700000000000001}]},{"osd":6,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0700000000000001,"15min":1.0860000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.2350000000000001,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0649999999999999,"15min":1.095},"min":{"1min":0.53800000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.2769999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.093}]}]},{"osd":6,"up_from":47,"seq":201863462964,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:50 2024","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.82099999999999995,"15min":0.83099999999999996},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.1759999999999999,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.92300000000000004},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.89900000000000002,"15min":0.89100000000000001},"min":{"1min":0.52300000000000002,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.82199999999999995}]},{"osd":1,"last update":"Sun Jul 28 19:28:50 2024","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.84899999999999998,"15min":0.84699999999999998},"min":{"1min":0.53200000000000003,"5min":0.503,"15min":0.503},"max":{"1min":1.2,"5min":1.48,"15min":1.48},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.876,"15min":0.85599999999999998},"min":{"1min":0.70499999999999996,"5min":0.499,"15min":0.499},"max":{"1min":1.238,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.70499999999999996}]},{"osd":2,"last update":"Sun Jul 28 19:28:50 2024","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.875,"15min":0.85499999999999998},"min":{"1min":0.64600000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0680000000000001,"5min":1.222,"15min":1.222},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.91800000000000004,"15min":0.89800000000000002},"min":{"1min":0.58199999999999996,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":1.038}]},{"osd":3,"last update":"Sun Jul 28 19:28:50 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.93400000000000005,"15min":0.92400000000000004},"min":{"1min":0.70499999999999996,"5min":0.498,"15min":0.498},"max":{"1min":1.236,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.01},{"interface":"front","average":{"1min":1.002,"5min":0.90000000000000002,"15min":0.86599999999999999},"min":{"1min":0.65100000000000002,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.232,"5min":1.298,"15min":1.298},"last":0.98699999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:28:50 2024","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.82899999999999996,"15min":0.80900000000000005},"min":{"1min":0.58699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.355,"5min":1.355,"15min":1.355},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.006,"5min":0.90200000000000002,"15min":0.82799999999999996},"min":{"1min":0.78300000000000003,"5min":0.316,"15min":0.316},"max":{"1min":1.274,"5min":1.617,"15min":1.617},"last":1.113}]},{"osd":5,"last update":"Sun Jul 28 19:28:50 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.89400000000000002,"15min":0.82599999999999996},"min":{"1min":0.70999999999999996,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.2589999999999999,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.91500000000000004,"15min":0.83399999999999996},"min":{"1min":0.747,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.4079999999999999,"5min":1.5449999999999999,"15min":1.5449999999999999},"last":0.85699999999999998}]},{"osd":7,"last update":"Sun Jul 28 19:28:36 2024","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.036,"15min":1.0509999999999999},"min":{"1min":0.44400000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.26,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0820000000000001,"15min":1.0900000000000001},"min":{"1min":0.46300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.4399999999999999,"5min":1.476,"15min":1.476},"last":1.1579999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724606,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.88600000000000001,"15min":0.85999999999999999},"min":{"1min":0.57199999999999995,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.258,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.995},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.84299999999999997,"15min":0.83799999999999997},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.2,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.72399999999999998}]},{"osd":1,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.88900000000000001,"15min":0.86399999999999999},"min":{"1min":0.61599999999999999,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.3089999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.89400000000000002,"15min":0.86599999999999999},"min":{"1min":0.64700000000000002,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.347,"5min":1.347,"15min":1.347},"last":0.76500000000000001}]},{"osd":2,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":0.871,"15min":0.86199999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.274,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.90300000000000002,"15min":0.88700000000000001},"min":{"1min":0.67800000000000005,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.2170000000000001,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.877}]},{"osd":3,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.91300000000000003,"15min":0.89100000000000001},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.397,"5min":1.397,"15min":1.397},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.92400000000000004,"15min":0.88200000000000001},"min":{"1min":0.621,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2849999999999999,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":0.79300000000000004}]},{"osd":4,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.77500000000000002,"15min":0.72299999999999998},"min":{"1min":0.42099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.81000000000000005,"15min":0.78100000000000003},"min":{"1min":0.434,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.4870000000000001,"5min":1.4870000000000001,"15min":1.4870000000000001},"last":0.59699999999999998}]},{"osd":6,"last update":"Sun Jul 28 19:28:49 2024","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.91200000000000003,"15min":0.85699999999999998},"min":{"1min":0.373,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.117,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.004,"5min":0.95199999999999996,"15min":0.91400000000000003},"min":{"1min":0.52300000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.24,"5min":2.1669999999999998,"15min":2.1669999999999998},"last":0.96199999999999997}]},{"osd":7,"last update":"Sun Jul 28 19:28:41 2024","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.008,"15min":1},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.306,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.028,"15min":1.038},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.254,"5min":1.516,"15min":1.516},"last":0.69599999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986247,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.86899999999999999,"15min":0.84299999999999997},"min":{"1min":0.56200000000000006,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.2529999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.83399999999999996,"15min":0.82199999999999995},"min":{"1min":0.57899999999999996,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.1299999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":1.0820000000000001}]},{"osd":1,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.82399999999999995,"15min":0.79200000000000004},"min":{"1min":0.51300000000000001,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.1000000000000001,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.88,"5min":0.88300000000000001,"15min":0.86299999999999999},"min":{"1min":0.57399999999999995,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2430000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.98299999999999998}]},{"osd":2,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.91800000000000004,"15min":0.88900000000000001},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.298,"5min":1.298,"15min":1.298},"last":1.054},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.877,"15min":0.86199999999999999},"min":{"1min":0.61899999999999999,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.78600000000000003}]},{"osd":3,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.89600000000000002,"15min":0.86799999999999999},"min":{"1min":0.52400000000000002,"5min":0.504,"15min":0.504},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.88600000000000001,"15min":0.86299999999999999},"min":{"1min":0.64400000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.173,"5min":1.583,"15min":1.583},"last":0.85999999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:27:57 2024","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.84199999999999997,"15min":0.79600000000000004},"min":{"1min":0.42899999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.5569999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.753},{"interface":"front","average":{"1min":1.008,"5min":0.85899999999999999,"15min":0.81699999999999995},"min":{"1min":0.50600000000000001,"5min":0.309,"15min":0.309},"max":{"1min":1.6919999999999999,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":1.105}]},{"osd":6,"last update":"Sun Jul 28 19:28:49 2024","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.92600000000000005,"15min":0.91000000000000003},"min":{"1min":0.67700000000000005,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.093,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":1.018,"5min":0.92000000000000004,"15min":0.85799999999999998},"min":{"1min":0.57699999999999996,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.1910000000000001,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.1679999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:28:37 2024","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.284,"15min":1.4139999999999999},"min":{"1min":0.47999999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.22,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.4099999999999999,"15min":1.5609999999999999},"min":{"1min":0.52300000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.202,"5min":10.920999999999999,"15min":10.920999999999999},"last":1.1379999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149777,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.73299999999999998,"15min":0.69299999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.999,"5min":1.909,"15min":1.909},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.81599999999999995,"15min":0.77700000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.3839999999999999,"5min":1.556,"15min":1.556},"last":0.69999999999999996}]},{"osd":1,"last update":"Sun Jul 28 19:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.73599999999999999,"15min":0.71199999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.476,"5min":1.583,"15min":1.583},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.71099999999999997,"15min":0.67300000000000004},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":0.64600000000000002}]},{"osd":2,"last update":"Sun Jul 28 19:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.79900000000000004,"15min":0.75900000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.839,"5min":1.839,"15min":1.839},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.751,"5min":0.74399999999999999,"15min":0.67700000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.3320000000000001,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.621}]},{"osd":4,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.89900000000000002,"15min":0.875},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.1419999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.024,"5min":1.0009999999999999,"15min":1.075},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.996,"5min":5.6539999999999999,"15min":5.6539999999999999},"last":0.88600000000000001}]},{"osd":5,"last update":"Sun Jul 28 19:28:00 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.93000000000000005,"15min":0.92000000000000004},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.208,"5min":1.341,"15min":1.341},"last":0.90600000000000003},{"interface":"front","average":{"1min":1.01,"5min":0.97799999999999998,"15min":0.96199999999999997},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2350000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.97699999999999998}]},{"osd":6,"last update":"Sun Jul 28 19:28:51 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0169999999999999,"15min":1.0249999999999999},"min":{"1min":0.58899999999999997,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":2.165,"5min":2.165,"15min":2.165},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.0089999999999999,"15min":1.006},"min":{"1min":0.72499999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.1990000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.113}]},{"osd":7,"last update":"Sun Jul 28 19:28:39 2024","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0760000000000001,"15min":1.089},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":2.0880000000000001,"5min":2.0880000000000001,"15min":2.0880000000000001},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.94099999999999995,"5min":1.0629999999999999,"15min":1.0780000000000001},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.28,"5min":1.401,"15min":1.401},"last":1.016}]}]},{"osd":2,"up_from":18,"seq":77309411422,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:30 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.63700000000000001,"15min":0.56699999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.82299999999999995,"5min":2.4569999999999999,"15min":2.4569999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.79900000000000004,"15min":0.68400000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.317},"max":{"1min":1.1299999999999999,"5min":1.456,"15min":1.456},"last":0.67800000000000005}]},{"osd":1,"last update":"Sun Jul 28 19:28:30 2024","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.70199999999999996,"15min":0.59299999999999997},"min":{"1min":0.51800000000000002,"5min":0.39800000000000002,"15min":0.30599999999999999},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.76600000000000001,"15min":0.65900000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":1.2230000000000001,"5min":1.7350000000000001,"15min":1.7350000000000001},"last":0.63600000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:28:24 2024","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.84499999999999997,"15min":0.75900000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.3759999999999999,"5min":1.9790000000000001,"15min":1.9790000000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.84799999999999998,"15min":0.80200000000000005},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.242,"5min":1.605,"15min":1.605},"last":0.77100000000000002}]},{"osd":4,"last update":"Sun Jul 28 19:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.88200000000000001,"15min":0.85299999999999998},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.085,"5min":1.196,"15min":1.196},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.96699999999999997,"15min":0.92300000000000004},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.95499999999999996}]},{"osd":5,"last update":"Sun Jul 28 19:28:02 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.98599999999999999,"15min":1.036},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.226,"5min":3.9969999999999999,"15min":3.9969999999999999},"last":1.026},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.0529999999999999,"15min":1.1339999999999999},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.504,"5min":5.3680000000000003,"15min":5.3680000000000003},"last":0.73199999999999998}]},{"osd":6,"last update":"Sun Jul 28 19:28:50 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.0109999999999999,"15min":1.0269999999999999},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.175,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.88},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0580000000000001,"15min":1.0509999999999999},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.234,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":1.0569999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:28:40 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.091,"15min":1.0980000000000001},"min":{"1min":0.751,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.319,"5min":1.766,"15min":1.766},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.02,"5min":1.0189999999999999,"15min":1.034},"min":{"1min":0.58699999999999997,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.331,"5min":1.454,"15min":1.454},"last":1.085}]}]},{"osd":1,"up_from":13,"seq":55834574951,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:28:42 2024","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.65400000000000003,"15min":0.55800000000000005},"min":{"1min":0.34100000000000003,"5min":0.33100000000000002,"15min":0.28399999999999997},"max":{"1min":2.2530000000000001,"5min":2.2530000000000001,"15min":2.2530000000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.72399999999999998,"15min":0.60399999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.1359999999999999,"5min":1.351,"15min":1.351},"last":0.62}]},{"osd":2,"last update":"Sun Jul 28 19:28:29 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.68300000000000005,"15min":0.59799999999999998},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.77700000000000002,"15min":0.69499999999999995},"min":{"1min":0.34100000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4039999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.78300000000000003}]},{"osd":3,"last update":"Sun Jul 28 19:28:22 2024","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.83899999999999997,"15min":0.78500000000000003},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.76200000000000001,"15min":0.748},"min":{"1min":0.32900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.92400000000000004,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.99199999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:28:10 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.97299999999999998,"15min":0.91700000000000004},"min":{"1min":0.69999999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.2789999999999999,"5min":1.538,"15min":1.538},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.91700000000000004,"15min":0.90200000000000002},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.53,"5min":1.53,"15min":1.53},"last":0.74099999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:28:02 2024","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.92300000000000004,"15min":0.90700000000000003},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.605,"5min":1.605,"15min":1.605},"last":0.93500000000000005},{"interface":"front","average":{"1min":1.018,"5min":1.0049999999999999,"15min":0.99099999999999999},"min":{"1min":0.75700000000000001,"5min":0.625,"15min":0.625},"max":{"1min":1.266,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.101}]},{"osd":6,"last update":"Sun Jul 28 19:28:51 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.079,"15min":1.0860000000000001},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":1.048},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0089999999999999,"15min":0.98099999999999998},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.778,"5min":1.778,"15min":1.778},"last":1.2170000000000001}]},{"osd":7,"last update":"Sun Jul 28 19:28:39 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.006,"15min":1.006},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.677,"5min":1.677,"15min":1.677},"last":1.286},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0649999999999999,"15min":1.089},"min":{"1min":0.60599999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":1.137}]}]},{"osd":0,"up_from":8,"seq":34359738481,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Jul 28 19:28:44 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.64800000000000002,"15min":0.55400000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.95699999999999996,"5min":1.0569999999999999,"15min":1.127},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.74099999999999999,"15min":0.61699999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.2869999999999999,"5min":1.347,"15min":1.347},"last":0.65900000000000003}]},{"osd":2,"last update":"Sun Jul 28 19:28:30 2024","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.77100000000000002,"15min":0.66800000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.32900000000000001},"max":{"1min":1.026,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.73499999999999999,"15min":0.67200000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29499999999999998},"max":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.68500000000000005}]},{"osd":3,"last update":"Sun Jul 28 19:28:30 2024","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.755,"15min":0.73399999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.002,"5min":1.28,"15min":1.28},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.83399999999999996,"15min":0.82799999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.1240000000000001,"5min":1.173,"15min":1.3220000000000001},"last":0.879}]},{"osd":4,"last update":"Sun Jul 28 19:28:08 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.90300000000000002,"15min":0.84299999999999997},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.93100000000000005,"15min":0.89300000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.292,"5min":1.327,"15min":1.327},"last":1.0920000000000001}]},{"osd":5,"last update":"Sun Jul 28 19:27:58 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.94999999999999996,"15min":0.94199999999999995},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.94399999999999995,"15min":0.92200000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3180000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":1.0389999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:27:55 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.022,"15min":1.028},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.94199999999999995},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.052,"15min":1.054},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.512,"5min":1.512,"15min":1.512},"last":1.0109999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:28:44 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.006,"15min":1.012},"min":{"1min":0.58999999999999997,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3129999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":1.151},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0289999999999999,"15min":1.0429999999999999},"min":{"1min":0.61899999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":1.1319999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-28T19:28:56.800 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-07-28T19:28:56.800 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-07-28T19:28:56.800 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-07-28T19:28:56.800 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:28:57.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:57 smithi100 bash[21828]: audit 2024-07-28T19:28:56.138194+0000 mgr.y (mgr.14391) 107 : audit [DBG] from='client.24497 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:28:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:57 smithi100 bash[31715]: audit 2024-07-28T19:28:56.138194+0000 mgr.y (mgr.14391) 107 : audit [DBG] from='client.24497 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:28:57.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:57 smithi138 bash[23952]: audit 2024-07-28T19:28:56.138194+0000 mgr.y (mgr.14391) 107 : audit [DBG] from='client.24497 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:28:58.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:28:58 smithi138 bash[23952]: cluster 2024-07-28T19:28:56.864210+0000 mgr.y (mgr.14391) 108 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:58.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:28:58 smithi100 bash[21828]: cluster 2024-07-28T19:28:56.864210+0000 mgr.y (mgr.14391) 108 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:28:58.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:28:58 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:28:58] "GET /metrics HTTP/1.1" 200 203263 "" "Prometheus/2.43.0" 2024-07-28T19:28:58.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:28:58 smithi100 bash[31715]: cluster 2024-07-28T19:28:56.864210+0000 mgr.y (mgr.14391) 108 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:00.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:00 smithi138 bash[23952]: cluster 2024-07-28T19:28:58.864765+0000 mgr.y (mgr.14391) 109 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:00.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:00 smithi100 bash[21828]: cluster 2024-07-28T19:28:58.864765+0000 mgr.y (mgr.14391) 109 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:00.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:00 smithi100 bash[31715]: cluster 2024-07-28T19:28:58.864765+0000 mgr.y (mgr.14391) 109 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:01.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:01 smithi138 bash[23952]: audit 2024-07-28T19:29:00.906703+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:01.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:01 smithi100 bash[21828]: audit 2024-07-28T19:29:00.906703+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:01.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:01 smithi100 bash[31715]: audit 2024-07-28T19:29:00.906703+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:01.580 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:29:02.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:02 smithi100 bash[21828]: cluster 2024-07-28T19:29:00.865396+0000 mgr.y (mgr.14391) 110 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:02.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:02 smithi100 bash[31715]: cluster 2024-07-28T19:29:00.865396+0000 mgr.y (mgr.14391) 110 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:02.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:02 smithi138 bash[23952]: cluster 2024-07-28T19:29:00.865396+0000 mgr.y (mgr.14391) 110 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:04.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:04 smithi100 bash[21828]: cluster 2024-07-28T19:29:02.865860+0000 mgr.y (mgr.14391) 111 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:04.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:04 smithi100 bash[31715]: cluster 2024-07-28T19:29:02.865860+0000 mgr.y (mgr.14391) 111 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:04.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:04 smithi138 bash[23952]: cluster 2024-07-28T19:29:02.865860+0000 mgr.y (mgr.14391) 111 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:04.868 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:29:04.868 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-07-28T19:29:05.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:05 smithi100 bash[21828]: audit 2024-07-28T19:29:04.875184+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.100:0/3565462599' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:29:05.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:05 smithi100 bash[31715]: audit 2024-07-28T19:29:04.875184+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.100:0/3565462599' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:29:05.551 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-07-28T19:29:05.551 INFO:tasks.cephadm:Setup complete, yielding 2024-07-28T19:29:05.552 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-07-28T19:29:05.561 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi100.front.sepia.ceph.com 2024-07-28T19:29:05.561 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-07-28T19:29:05.562 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:05 smithi138 bash[23952]: audit 2024-07-28T19:29:04.875184+0000 mon.c (mon.2) 90 : audit [DBG] from='client.? 172.21.15.100:0/3565462599' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:29:06.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:06 smithi100 bash[21828]: cluster 2024-07-28T19:29:04.866355+0000 mgr.y (mgr.14391) 112 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:06 smithi100 bash[31715]: cluster 2024-07-28T19:29:04.866355+0000 mgr.y (mgr.14391) 112 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:06.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:06 smithi138 bash[23952]: cluster 2024-07-28T19:29:04.866355+0000 mgr.y (mgr.14391) 112 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:06.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:29:06 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:29:08.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:08 smithi100 bash[21828]: cluster 2024-07-28T19:29:06.866988+0000 mgr.y (mgr.14391) 113 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:08.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:29:08 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:08] "GET /metrics HTTP/1.1" 200 203256 "" "Prometheus/2.43.0" 2024-07-28T19:29:08.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:08 smithi100 bash[31715]: cluster 2024-07-28T19:29:06.866988+0000 mgr.y (mgr.14391) 113 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:08.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:08 smithi138 bash[23952]: cluster 2024-07-28T19:29:06.866988+0000 mgr.y (mgr.14391) 113 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:10.340 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:29:10.466 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:10 smithi100 bash[21828]: cluster 2024-07-28T19:29:08.867598+0000 mgr.y (mgr.14391) 114 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:10.467 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:10 smithi100 bash[31715]: cluster 2024-07-28T19:29:08.867598+0000 mgr.y (mgr.14391) 114 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:10.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:10 smithi138 bash[23952]: cluster 2024-07-28T19:29:08.867598+0000 mgr.y (mgr.14391) 114 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:12.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:12 smithi100 bash[21828]: cluster 2024-07-28T19:29:10.868152+0000 mgr.y (mgr.14391) 115 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:12 smithi100 bash[31715]: cluster 2024-07-28T19:29:10.868152+0000 mgr.y (mgr.14391) 115 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:12.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:12 smithi138 bash[23952]: cluster 2024-07-28T19:29:10.868152+0000 mgr.y (mgr.14391) 115 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:14.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:14 smithi100 bash[21828]: cluster 2024-07-28T19:29:12.868813+0000 mgr.y (mgr.14391) 116 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:14.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:14 smithi100 bash[31715]: cluster 2024-07-28T19:29:12.868813+0000 mgr.y (mgr.14391) 116 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:14.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:14 smithi138 bash[23952]: cluster 2024-07-28T19:29:12.868813+0000 mgr.y (mgr.14391) 116 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:15.484 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[21828]: audit 2024-07-28T19:29:13.743981+0000 mgr.y (mgr.14391) 117 : audit [DBG] from='client.24509 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[21828]: audit 2024-07-28T19:29:13.744933+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[21828]: audit 2024-07-28T19:29:13.745543+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[21828]: audit 2024-07-28T19:29:14.231136+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[21828]: cluster 2024-07-28T19:29:14.231248+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[21828]: audit 2024-07-28T19:29:14.237641+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[21828]: audit 2024-07-28T19:29:14.238220+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[21828]: debug 2024-07-28T19:29:15.251+0000 7f9af9d53640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[31715]: audit 2024-07-28T19:29:13.743981+0000 mgr.y (mgr.14391) 117 : audit [DBG] from='client.24509 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[31715]: audit 2024-07-28T19:29:13.744933+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-28T19:29:15.485 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[31715]: audit 2024-07-28T19:29:13.745543+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-28T19:29:15.486 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[31715]: audit 2024-07-28T19:29:14.231136+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-07-28T19:29:15.486 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[31715]: cluster 2024-07-28T19:29:14.231248+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-07-28T19:29:15.486 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[31715]: audit 2024-07-28T19:29:14.237641+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:15.486 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:15 smithi100 bash[31715]: audit 2024-07-28T19:29:14.238220+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:15.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:15 smithi138 bash[23952]: audit 2024-07-28T19:29:13.743981+0000 mgr.y (mgr.14391) 117 : audit [DBG] from='client.24509 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:29:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:15 smithi138 bash[23952]: audit 2024-07-28T19:29:13.744933+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-28T19:29:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:15 smithi138 bash[23952]: audit 2024-07-28T19:29:13.745543+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-07-28T19:29:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:15 smithi138 bash[23952]: audit 2024-07-28T19:29:14.231136+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-07-28T19:29:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:15 smithi138 bash[23952]: cluster 2024-07-28T19:29:14.231248+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-07-28T19:29:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:15 smithi138 bash[23952]: audit 2024-07-28T19:29:14.237641+0000 mon.c (mon.2) 92 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:15 smithi138 bash[23952]: audit 2024-07-28T19:29:14.238220+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:16.279 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cluster 2024-07-28T19:29:14.869432+0000 mgr.y (mgr.14391) 118 : cluster [DBG] pgmap v90: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:16.279 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cluster 2024-07-28T19:29:15.226510+0000 mon.a (mon.0) 736 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:29:16.279 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.242806+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-07-28T19:29:16.279 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cluster 2024-07-28T19:29:15.243118+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-07-28T19:29:16.279 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.252548+0000 mon.c (mon.2) 93 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:16.279 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.254386+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:16.279 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cluster 2024-07-28T19:29:15.256122+0000 mon.a (mon.0) 740 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cluster 2024-07-28T19:29:15.256196+0000 mon.a (mon.0) 741 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cluster 2024-07-28T19:29:15.271880+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.273696+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cluster 2024-07-28T19:29:15.273882+0000 mon.a (mon.0) 744 : cluster [DBG] fsmap foo:0 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cephadm 2024-07-28T19:29:15.278856+0000 mgr.y (mgr.14391) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.290206+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.292352+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.295316+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:16.280 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.296716+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:16.281 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.309710+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.281 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.314169+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:16.281 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.314512+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:16.281 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.318329+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-28T19:29:16.281 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.319234+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:16.281 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: cephadm 2024-07-28T19:29:15.320085+0000 mgr.y (mgr.14391) 120 : cephadm [INF] Deploying daemon mds.foo.smithi100.ppcqpk on smithi100 2024-07-28T19:29:16.281 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.914540+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.282 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[21828]: audit 2024-07-28T19:29:15.916489+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:16.283 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-07-28T19:29:16.284 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cluster 2024-07-28T19:29:14.869432+0000 mgr.y (mgr.14391) 118 : cluster [DBG] pgmap v90: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:16.284 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cluster 2024-07-28T19:29:15.226510+0000 mon.a (mon.0) 736 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:29:16.284 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.242806+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-07-28T19:29:16.285 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cluster 2024-07-28T19:29:15.243118+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-07-28T19:29:16.285 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.252548+0000 mon.c (mon.2) 93 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:16.285 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.254386+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:16.285 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cluster 2024-07-28T19:29:15.256122+0000 mon.a (mon.0) 740 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-07-28T19:29:16.285 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cluster 2024-07-28T19:29:15.256196+0000 mon.a (mon.0) 741 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-07-28T19:29:16.285 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cluster 2024-07-28T19:29:15.271880+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-07-28T19:29:16.285 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.273696+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-07-28T19:29:16.286 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cluster 2024-07-28T19:29:15.273882+0000 mon.a (mon.0) 744 : cluster [DBG] fsmap foo:0 2024-07-28T19:29:16.286 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cephadm 2024-07-28T19:29:15.278856+0000 mgr.y (mgr.14391) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-07-28T19:29:16.286 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.290206+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.286 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.292352+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:16.286 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.295316+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:16.286 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.296716+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:16.287 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.309710+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.287 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.314169+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:16.287 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.314512+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:16.287 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.318329+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-28T19:29:16.287 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.319234+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:16.287 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: cephadm 2024-07-28T19:29:15.320085+0000 mgr.y (mgr.14391) 120 : cephadm [INF] Deploying daemon mds.foo.smithi100.ppcqpk on smithi100 2024-07-28T19:29:16.287 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.914540+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.287 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:16 smithi100 bash[31715]: audit 2024-07-28T19:29:15.916489+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:16.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:29:16.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cluster 2024-07-28T19:29:14.869432+0000 mgr.y (mgr.14391) 118 : cluster [DBG] pgmap v90: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:16.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cluster 2024-07-28T19:29:15.226510+0000 mon.a (mon.0) 736 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:29:16.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.242806+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-07-28T19:29:16.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cluster 2024-07-28T19:29:15.243118+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.252548+0000 mon.c (mon.2) 93 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.254386+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cluster 2024-07-28T19:29:15.256122+0000 mon.a (mon.0) 740 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cluster 2024-07-28T19:29:15.256196+0000 mon.a (mon.0) 741 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cluster 2024-07-28T19:29:15.271880+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.273696+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cluster 2024-07-28T19:29:15.273882+0000 mon.a (mon.0) 744 : cluster [DBG] fsmap foo:0 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cephadm 2024-07-28T19:29:15.278856+0000 mgr.y (mgr.14391) 119 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.290206+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.292352+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.295316+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.296716+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.309710+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.314169+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:16.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.314512+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:16.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.318329+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-28T19:29:16.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.319234+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:16.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: cephadm 2024-07-28T19:29:15.320085+0000 mgr.y (mgr.14391) 120 : cephadm [INF] Deploying daemon mds.foo.smithi100.ppcqpk on smithi100 2024-07-28T19:29:16.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.914540+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:16.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:16 smithi138 bash[23952]: audit 2024-07-28T19:29:15.916489+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:17.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:17 smithi138 bash[23952]: cluster 2024-07-28T19:29:16.271932+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-07-28T19:29:17.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:17 smithi100 bash[21828]: cluster 2024-07-28T19:29:16.271932+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-07-28T19:29:17.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:17 smithi100 bash[31715]: cluster 2024-07-28T19:29:16.271932+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-07-28T19:29:18.111 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.111 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.112 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.112 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.112 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.113 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.113 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:29:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.113 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.114 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.366 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.366 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.367 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.367 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.367 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.367 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.367 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:18 smithi100 bash[21828]: cluster 2024-07-28T19:29:16.870316+0000 mgr.y (mgr.14391) 121 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:18.368 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:18 smithi100 bash[21828]: cluster 2024-07-28T19:29:17.264416+0000 mon.a (mon.0) 751 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:29:18.368 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:18 smithi100 bash[21828]: cluster 2024-07-28T19:29:17.286157+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-07-28T19:29:18.368 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.368 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:29:18 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:18] "GET /metrics HTTP/1.1" 200 208052 "" "Prometheus/2.43.0" 2024-07-28T19:29:18.368 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.368 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:18.369 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:18 smithi100 bash[31715]: cluster 2024-07-28T19:29:16.870316+0000 mgr.y (mgr.14391) 121 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:18.369 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:18 smithi100 bash[31715]: cluster 2024-07-28T19:29:17.264416+0000 mon.a (mon.0) 751 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:29:18.369 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:18 smithi100 bash[31715]: cluster 2024-07-28T19:29:17.286157+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-07-28T19:29:18.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:18 smithi138 bash[23952]: cluster 2024-07-28T19:29:16.870316+0000 mgr.y (mgr.14391) 121 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:18.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:18 smithi138 bash[23952]: cluster 2024-07-28T19:29:17.264416+0000 mon.a (mon.0) 751 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:29:18.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:18 smithi138 bash[23952]: cluster 2024-07-28T19:29:17.286157+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[31715]: audit 2024-07-28T19:29:18.404064+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[31715]: audit 2024-07-28T19:29:18.411046+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[31715]: audit 2024-07-28T19:29:18.417086+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[31715]: audit 2024-07-28T19:29:18.418261+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[31715]: audit 2024-07-28T19:29:18.418602+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[31715]: audit 2024-07-28T19:29:18.422057+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[31715]: audit 2024-07-28T19:29:18.423657+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[31715]: cephadm 2024-07-28T19:29:18.424473+0000 mgr.y (mgr.14391) 122 : cephadm [INF] Deploying daemon mds.foo.smithi138.nphkis on smithi138 2024-07-28T19:29:19.670 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[21828]: audit 2024-07-28T19:29:18.404064+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.671 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[21828]: audit 2024-07-28T19:29:18.411046+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.671 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[21828]: audit 2024-07-28T19:29:18.417086+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.671 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[21828]: audit 2024-07-28T19:29:18.418261+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:19.671 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[21828]: audit 2024-07-28T19:29:18.418602+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:19.671 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[21828]: audit 2024-07-28T19:29:18.422057+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-28T19:29:19.671 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[21828]: audit 2024-07-28T19:29:18.423657+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:19.671 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:19 smithi100 bash[21828]: cephadm 2024-07-28T19:29:18.424473+0000 mgr.y (mgr.14391) 122 : cephadm [INF] Deploying daemon mds.foo.smithi138.nphkis on smithi138 2024-07-28T19:29:19.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:19 smithi138 bash[23952]: audit 2024-07-28T19:29:18.404064+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:19 smithi138 bash[23952]: audit 2024-07-28T19:29:18.411046+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:19 smithi138 bash[23952]: audit 2024-07-28T19:29:18.417086+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:19.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:19 smithi138 bash[23952]: audit 2024-07-28T19:29:18.418261+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:19.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:19 smithi138 bash[23952]: audit 2024-07-28T19:29:18.418602+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:29:19.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:19 smithi138 bash[23952]: audit 2024-07-28T19:29:18.422057+0000 mon.a (mon.0) 757 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-07-28T19:29:19.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:19 smithi138 bash[23952]: audit 2024-07-28T19:29:18.423657+0000 mon.c (mon.2) 101 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:19.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:19 smithi138 bash[23952]: cephadm 2024-07-28T19:29:18.424473+0000 mgr.y (mgr.14391) 122 : cephadm [INF] Deploying daemon mds.foo.smithi138.nphkis on smithi138 2024-07-28T19:29:20.681 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:20 smithi138 bash[23952]: cluster 2024-07-28T19:29:18.870917+0000 mgr.y (mgr.14391) 123 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:20.681 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.682 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.682 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.682 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.683 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.683 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.683 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.683 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.684 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.743 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:20 smithi100 bash[21828]: cluster 2024-07-28T19:29:18.870917+0000 mgr.y (mgr.14391) 123 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:20.743 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:20 smithi100 bash[31715]: cluster 2024-07-28T19:29:18.870917+0000 mgr.y (mgr.14391) 123 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:20.938 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.939 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.939 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.939 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.940 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.940 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.940 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.940 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:20.941 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:29:20 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:29:21.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.429888+0000 mon.a (mon.0) 758 : cluster [DBG] mds.? [v2:172.21.15.100:6834/4040051651,v1:172.21.15.100:6835/4040051651] up:boot 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.429997+0000 mon.a (mon.0) 759 : cluster [INF] daemon mds.foo.smithi100.ppcqpk assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.430115+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.430155+0000 mon.a (mon.0) 761 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.430189+0000 mon.a (mon.0) 762 : cluster [INF] Cluster is now healthy 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.430585+0000 mon.a (mon.0) 763 : cluster [DBG] fsmap foo:0 1 up:standby 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: audit 2024-07-28T19:29:20.432483+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.438808+0000 mon.a (mon.0) 764 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:creating} 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.464413+0000 mon.a (mon.0) 765 : cluster [INF] daemon mds.foo.smithi100.ppcqpk is now active in filesystem foo as rank 0 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: audit 2024-07-28T19:29:20.987316+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: audit 2024-07-28T19:29:21.002686+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: audit 2024-07-28T19:29:21.014740+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: audit 2024-07-28T19:29:21.026353+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: audit 2024-07-28T19:29:21.036754+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[21828]: audit 2024-07-28T19:29:21.093781+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.429888+0000 mon.a (mon.0) 758 : cluster [DBG] mds.? [v2:172.21.15.100:6834/4040051651,v1:172.21.15.100:6835/4040051651] up:boot 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.429997+0000 mon.a (mon.0) 759 : cluster [INF] daemon mds.foo.smithi100.ppcqpk assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.430115+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.430155+0000 mon.a (mon.0) 761 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.430189+0000 mon.a (mon.0) 762 : cluster [INF] Cluster is now healthy 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.430585+0000 mon.a (mon.0) 763 : cluster [DBG] fsmap foo:0 1 up:standby 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: audit 2024-07-28T19:29:20.432483+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.438808+0000 mon.a (mon.0) 764 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:creating} 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.464413+0000 mon.a (mon.0) 765 : cluster [INF] daemon mds.foo.smithi100.ppcqpk is now active in filesystem foo as rank 0 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: audit 2024-07-28T19:29:20.987316+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: audit 2024-07-28T19:29:21.002686+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: audit 2024-07-28T19:29:21.014740+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: audit 2024-07-28T19:29:21.026353+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: audit 2024-07-28T19:29:21.036754+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:21 smithi100 bash[31715]: audit 2024-07-28T19:29:21.093781+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.429888+0000 mon.a (mon.0) 758 : cluster [DBG] mds.? [v2:172.21.15.100:6834/4040051651,v1:172.21.15.100:6835/4040051651] up:boot 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.429997+0000 mon.a (mon.0) 759 : cluster [INF] daemon mds.foo.smithi100.ppcqpk assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.430115+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.430155+0000 mon.a (mon.0) 761 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.430189+0000 mon.a (mon.0) 762 : cluster [INF] Cluster is now healthy 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.430585+0000 mon.a (mon.0) 763 : cluster [DBG] fsmap foo:0 1 up:standby 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: audit 2024-07-28T19:29:20.432483+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.438808+0000 mon.a (mon.0) 764 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:creating} 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.464413+0000 mon.a (mon.0) 765 : cluster [INF] daemon mds.foo.smithi100.ppcqpk is now active in filesystem foo as rank 0 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: audit 2024-07-28T19:29:20.987316+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: audit 2024-07-28T19:29:21.002686+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: audit 2024-07-28T19:29:21.014740+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: audit 2024-07-28T19:29:21.026353+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: audit 2024-07-28T19:29:21.036754+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:21 smithi138 bash[23952]: audit 2024-07-28T19:29:21.093781+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:22.080 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:29:22.743 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:22 smithi100 bash[31715]: cluster 2024-07-28T19:29:20.871565+0000 mgr.y (mgr.14391) 124 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:22.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:22 smithi100 bash[31715]: cluster 2024-07-28T19:29:21.448860+0000 mon.a (mon.0) 771 : cluster [DBG] mds.? [v2:172.21.15.100:6834/4040051651,v1:172.21.15.100:6835/4040051651] up:active 2024-07-28T19:29:22.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:22 smithi100 bash[31715]: cluster 2024-07-28T19:29:21.448936+0000 mon.a (mon.0) 772 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:active} 2024-07-28T19:29:22.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:22 smithi100 bash[21828]: cluster 2024-07-28T19:29:20.871565+0000 mgr.y (mgr.14391) 124 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:22.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:22 smithi100 bash[21828]: cluster 2024-07-28T19:29:21.448860+0000 mon.a (mon.0) 771 : cluster [DBG] mds.? [v2:172.21.15.100:6834/4040051651,v1:172.21.15.100:6835/4040051651] up:active 2024-07-28T19:29:22.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:22 smithi100 bash[21828]: cluster 2024-07-28T19:29:21.448936+0000 mon.a (mon.0) 772 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:active} 2024-07-28T19:29:22.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:22 smithi138 bash[23952]: cluster 2024-07-28T19:29:20.871565+0000 mgr.y (mgr.14391) 124 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:22.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:22 smithi138 bash[23952]: cluster 2024-07-28T19:29:21.448860+0000 mon.a (mon.0) 771 : cluster [DBG] mds.? [v2:172.21.15.100:6834/4040051651,v1:172.21.15.100:6835/4040051651] up:active 2024-07-28T19:29:22.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:22 smithi138 bash[23952]: cluster 2024-07-28T19:29:21.448936+0000 mon.a (mon.0) 772 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:active} 2024-07-28T19:29:23.617 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.449176+0000 mon.a (mon.0) 773 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:boot 2024-07-28T19:29:23.617 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.449290+0000 mon.a (mon.0) 774 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi100.ppcqpk in favor of higher affinity standby. 2024-07-28T19:29:23.617 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.449345+0000 mon.a (mon.0) 775 : cluster [WRN] Replacing daemon mds.foo.smithi100.ppcqpk as rank 0 with standby daemon mds.foo.smithi138.nphkis 2024-07-28T19:29:23.617 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.449935+0000 mon.a (mon.0) 776 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-07-28T19:29:23.617 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.449976+0000 mon.a (mon.0) 777 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-07-28T19:29:23.617 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.450517+0000 mon.a (mon.0) 778 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:active} 1 up:standby 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: audit 2024-07-28T19:29:22.452338+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.462003+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.462124+0000 mon.a (mon.0) 780 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:replay} 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.449176+0000 mon.a (mon.0) 773 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:boot 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.449290+0000 mon.a (mon.0) 774 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi100.ppcqpk in favor of higher affinity standby. 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.449345+0000 mon.a (mon.0) 775 : cluster [WRN] Replacing daemon mds.foo.smithi100.ppcqpk as rank 0 with standby daemon mds.foo.smithi138.nphkis 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.449935+0000 mon.a (mon.0) 776 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.449976+0000 mon.a (mon.0) 777 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.450517+0000 mon.a (mon.0) 778 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:active} 1 up:standby 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: audit 2024-07-28T19:29:22.452338+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.462003+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-07-28T19:29:23.618 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:23 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.462124+0000 mon.a (mon.0) 780 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:replay} 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.449176+0000 mon.a (mon.0) 773 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:boot 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.449290+0000 mon.a (mon.0) 774 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi100.ppcqpk in favor of higher affinity standby. 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.449345+0000 mon.a (mon.0) 775 : cluster [WRN] Replacing daemon mds.foo.smithi100.ppcqpk as rank 0 with standby daemon mds.foo.smithi138.nphkis 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.449935+0000 mon.a (mon.0) 776 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.449976+0000 mon.a (mon.0) 777 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.450517+0000 mon.a (mon.0) 778 : cluster [DBG] fsmap foo:1 {0=foo.smithi100.ppcqpk=up:active} 1 up:standby 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: audit 2024-07-28T19:29:22.452338+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.462003+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-07-28T19:29:23.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:23 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.462124+0000 mon.a (mon.0) 780 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:replay} 2024-07-28T19:29:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[21828]: cluster 2024-07-28T19:29:22.872191+0000 mgr.y (mgr.14391) 125 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-07-28T19:29:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[21828]: cluster 2024-07-28T19:29:23.453642+0000 mon.a (mon.0) 781 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-07-28T19:29:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[21828]: audit 2024-07-28T19:29:23.464344+0000 mon.c (mon.2) 105 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:29:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[21828]: cluster 2024-07-28T19:29:23.464918+0000 mon.a (mon.0) 782 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:reconnect 2024-07-28T19:29:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[21828]: cluster 2024-07-28T19:29:23.464947+0000 mon.a (mon.0) 783 : cluster [DBG] mds.? [v2:172.21.15.100:6834/1539587085,v1:172.21.15.100:6835/1539587085] up:boot 2024-07-28T19:29:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[21828]: cluster 2024-07-28T19:29:23.465011+0000 mon.a (mon.0) 784 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:reconnect} 1 up:standby 2024-07-28T19:29:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[31715]: cluster 2024-07-28T19:29:22.872191+0000 mgr.y (mgr.14391) 125 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-07-28T19:29:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[31715]: cluster 2024-07-28T19:29:23.453642+0000 mon.a (mon.0) 781 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-07-28T19:29:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[31715]: audit 2024-07-28T19:29:23.464344+0000 mon.c (mon.2) 105 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:29:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[31715]: cluster 2024-07-28T19:29:23.464918+0000 mon.a (mon.0) 782 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:reconnect 2024-07-28T19:29:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[31715]: cluster 2024-07-28T19:29:23.464947+0000 mon.a (mon.0) 783 : cluster [DBG] mds.? [v2:172.21.15.100:6834/1539587085,v1:172.21.15.100:6835/1539587085] up:boot 2024-07-28T19:29:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:24 smithi100 bash[31715]: cluster 2024-07-28T19:29:23.465011+0000 mon.a (mon.0) 784 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:reconnect} 1 up:standby 2024-07-28T19:29:24.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:24 smithi138 bash[23952]: cluster 2024-07-28T19:29:22.872191+0000 mgr.y (mgr.14391) 125 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s wr, 6 op/s 2024-07-28T19:29:24.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:24 smithi138 bash[23952]: cluster 2024-07-28T19:29:23.453642+0000 mon.a (mon.0) 781 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-07-28T19:29:24.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:24 smithi138 bash[23952]: audit 2024-07-28T19:29:23.464344+0000 mon.c (mon.2) 105 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:29:24.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:24 smithi138 bash[23952]: cluster 2024-07-28T19:29:23.464918+0000 mon.a (mon.0) 782 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:reconnect 2024-07-28T19:29:24.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:24 smithi138 bash[23952]: cluster 2024-07-28T19:29:23.464947+0000 mon.a (mon.0) 783 : cluster [DBG] mds.? [v2:172.21.15.100:6834/1539587085,v1:172.21.15.100:6835/1539587085] up:boot 2024-07-28T19:29:24.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:24 smithi138 bash[23952]: cluster 2024-07-28T19:29:23.465011+0000 mon.a (mon.0) 784 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:reconnect} 1 up:standby 2024-07-28T19:29:25.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:25 smithi100 bash[21828]: cluster 2024-07-28T19:29:24.476096+0000 mon.a (mon.0) 785 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:rejoin 2024-07-28T19:29:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:25 smithi100 bash[21828]: cluster 2024-07-28T19:29:24.476263+0000 mon.a (mon.0) 786 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:rejoin} 1 up:standby 2024-07-28T19:29:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:25 smithi100 bash[21828]: cluster 2024-07-28T19:29:24.476828+0000 mon.a (mon.0) 787 : cluster [INF] daemon mds.foo.smithi138.nphkis is now active in filesystem foo as rank 0 2024-07-28T19:29:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:25 smithi100 bash[31715]: cluster 2024-07-28T19:29:24.476096+0000 mon.a (mon.0) 785 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:rejoin 2024-07-28T19:29:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:25 smithi100 bash[31715]: cluster 2024-07-28T19:29:24.476263+0000 mon.a (mon.0) 786 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:rejoin} 1 up:standby 2024-07-28T19:29:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:25 smithi100 bash[31715]: cluster 2024-07-28T19:29:24.476828+0000 mon.a (mon.0) 787 : cluster [INF] daemon mds.foo.smithi138.nphkis is now active in filesystem foo as rank 0 2024-07-28T19:29:25.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:25 smithi138 bash[23952]: cluster 2024-07-28T19:29:24.476096+0000 mon.a (mon.0) 785 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:rejoin 2024-07-28T19:29:25.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:25 smithi138 bash[23952]: cluster 2024-07-28T19:29:24.476263+0000 mon.a (mon.0) 786 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi138.nphkis=up:rejoin} 1 up:standby 2024-07-28T19:29:25.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:25 smithi138 bash[23952]: cluster 2024-07-28T19:29:24.476828+0000 mon.a (mon.0) 787 : cluster [INF] daemon mds.foo.smithi138.nphkis is now active in filesystem foo as rank 0 2024-07-28T19:29:26.264 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-07-28T19:29:26.475 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:29:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[21828]: cluster 2024-07-28T19:29:24.872833+0000 mgr.y (mgr.14391) 126 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-07-28T19:29:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[21828]: cluster 2024-07-28T19:29:25.471751+0000 mon.a (mon.0) 788 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-07-28T19:29:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[21828]: cluster 2024-07-28T19:29:25.471774+0000 mon.a (mon.0) 789 : cluster [INF] Cluster is now healthy 2024-07-28T19:29:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[21828]: cluster 2024-07-28T19:29:25.482219+0000 mon.a (mon.0) 790 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:active 2024-07-28T19:29:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[21828]: cluster 2024-07-28T19:29:25.482308+0000 mon.a (mon.0) 791 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:29:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[21828]: audit 2024-07-28T19:29:25.696966+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[21828]: audit 2024-07-28T19:29:25.706924+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[21828]: audit 2024-07-28T19:29:26.014188+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[31715]: cluster 2024-07-28T19:29:24.872833+0000 mgr.y (mgr.14391) 126 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-07-28T19:29:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[31715]: cluster 2024-07-28T19:29:25.471751+0000 mon.a (mon.0) 788 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-07-28T19:29:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[31715]: cluster 2024-07-28T19:29:25.471774+0000 mon.a (mon.0) 789 : cluster [INF] Cluster is now healthy 2024-07-28T19:29:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[31715]: cluster 2024-07-28T19:29:25.482219+0000 mon.a (mon.0) 790 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:active 2024-07-28T19:29:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[31715]: cluster 2024-07-28T19:29:25.482308+0000 mon.a (mon.0) 791 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:29:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[31715]: audit 2024-07-28T19:29:25.696966+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[31715]: audit 2024-07-28T19:29:25.706924+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:26 smithi100 bash[31715]: audit 2024-07-28T19:29:26.014188+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:26.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[23952]: cluster 2024-07-28T19:29:24.872833+0000 mgr.y (mgr.14391) 126 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-07-28T19:29:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[23952]: cluster 2024-07-28T19:29:25.471751+0000 mon.a (mon.0) 788 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-07-28T19:29:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[23952]: cluster 2024-07-28T19:29:25.471774+0000 mon.a (mon.0) 789 : cluster [INF] Cluster is now healthy 2024-07-28T19:29:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[23952]: cluster 2024-07-28T19:29:25.482219+0000 mon.a (mon.0) 790 : cluster [DBG] mds.? [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] up:active 2024-07-28T19:29:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[23952]: cluster 2024-07-28T19:29:25.482308+0000 mon.a (mon.0) 791 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:29:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[23952]: audit 2024-07-28T19:29:25.696966+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[23952]: audit 2024-07-28T19:29:25.706924+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:26.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:26 smithi138 bash[23952]: audit 2024-07-28T19:29:26.014188+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.701913+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.716486+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.719830+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:27.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.721683+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.732338+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.802027+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.805224+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.806810+0000 mon.c (mon.2) 110 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: audit 2024-07-28T19:29:26.815360+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[21828]: cluster 2024-07-28T19:29:26.873896+0000 mgr.y (mgr.14391) 127 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.701913+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.716486+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.719830+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.721683+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.732338+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.802027+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:27.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.805224+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:27.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.806810+0000 mon.c (mon.2) 110 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:27.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: audit 2024-07-28T19:29:26.815360+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:27.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:27 smithi100 bash[31715]: cluster 2024-07-28T19:29:26.873896+0000 mgr.y (mgr.14391) 127 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-07-28T19:29:28.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.701913+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.716486+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.719830+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.721683+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.732338+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.802027+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.805224+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.806810+0000 mon.c (mon.2) 110 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: audit 2024-07-28T19:29:26.815360+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:28.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:27 smithi138 bash[23952]: cluster 2024-07-28T19:29:26.873896+0000 mgr.y (mgr.14391) 127 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-07-28T19:29:28.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:29:28 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:28] "GET /metrics HTTP/1.1" 200 208052 "" "Prometheus/2.43.0" 2024-07-28T19:29:30.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:29 smithi100 bash[21828]: cluster 2024-07-28T19:29:28.874548+0000 mgr.y (mgr.14391) 128 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-28T19:29:30.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:29 smithi100 bash[31715]: cluster 2024-07-28T19:29:28.874548+0000 mgr.y (mgr.14391) 128 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-28T19:29:30.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:29 smithi138 bash[23952]: cluster 2024-07-28T19:29:28.874548+0000 mgr.y (mgr.14391) 128 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-28T19:29:31.051 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:29:31.194 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[31715]: audit 2024-07-28T19:29:30.862411+0000 mon.c (mon.2) 111 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.194 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[31715]: audit 2024-07-28T19:29:30.863123+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-07-28T19:29:31.194 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[31715]: audit 2024-07-28T19:29:30.863355+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.194 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[31715]: audit 2024-07-28T19:29:30.863740+0000 mon.c (mon.2) 113 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.194 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[31715]: audit 2024-07-28T19:29:30.864083+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-07-28T19:29:31.194 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[31715]: audit 2024-07-28T19:29:30.864617+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[31715]: audit 2024-07-28T19:29:30.916194+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[31715]: audit 2024-07-28T19:29:30.922662+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[21828]: audit 2024-07-28T19:29:30.862411+0000 mon.c (mon.2) 111 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[21828]: audit 2024-07-28T19:29:30.863123+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[21828]: audit 2024-07-28T19:29:30.863355+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[21828]: audit 2024-07-28T19:29:30.863740+0000 mon.c (mon.2) 113 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[21828]: audit 2024-07-28T19:29:30.864083+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[21828]: audit 2024-07-28T19:29:30.864617+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[21828]: audit 2024-07-28T19:29:30.916194+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:31.195 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:30 smithi100 bash[21828]: audit 2024-07-28T19:29:30.922662+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:31.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:30 smithi138 bash[23952]: audit 2024-07-28T19:29:30.862411+0000 mon.c (mon.2) 111 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:30 smithi138 bash[23952]: audit 2024-07-28T19:29:30.863123+0000 mon.c (mon.2) 112 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-07-28T19:29:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:30 smithi138 bash[23952]: audit 2024-07-28T19:29:30.863355+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:30 smithi138 bash[23952]: audit 2024-07-28T19:29:30.863740+0000 mon.c (mon.2) 113 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:30 smithi138 bash[23952]: audit 2024-07-28T19:29:30.864083+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-07-28T19:29:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:30 smithi138 bash[23952]: audit 2024-07-28T19:29:30.864617+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-07-28T19:29:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:30 smithi138 bash[23952]: audit 2024-07-28T19:29:30.916194+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:29:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:30 smithi138 bash[23952]: audit 2024-07-28T19:29:30.922662+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[21828]: cluster 2024-07-28T19:29:30.875176+0000 mgr.y (mgr.14391) 129 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-28T19:29:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[21828]: audit 2024-07-28T19:29:30.947863+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-07-28T19:29:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[21828]: audit 2024-07-28T19:29:30.948110+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-07-28T19:29:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[21828]: audit 2024-07-28T19:29:30.948294+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-07-28T19:29:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[21828]: cluster 2024-07-28T19:29:30.948443+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-07-28T19:29:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[31715]: cluster 2024-07-28T19:29:30.875176+0000 mgr.y (mgr.14391) 129 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-28T19:29:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[31715]: audit 2024-07-28T19:29:30.947863+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-07-28T19:29:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[31715]: audit 2024-07-28T19:29:30.948110+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-07-28T19:29:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[31715]: audit 2024-07-28T19:29:30.948294+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-07-28T19:29:32.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:31 smithi100 bash[31715]: cluster 2024-07-28T19:29:30.948443+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-07-28T19:29:32.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:31 smithi138 bash[23952]: cluster 2024-07-28T19:29:30.875176+0000 mgr.y (mgr.14391) 129 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-07-28T19:29:32.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:31 smithi138 bash[23952]: audit 2024-07-28T19:29:30.947863+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-07-28T19:29:32.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:31 smithi138 bash[23952]: audit 2024-07-28T19:29:30.948110+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-07-28T19:29:32.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:31 smithi138 bash[23952]: audit 2024-07-28T19:29:30.948294+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-07-28T19:29:32.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:31 smithi138 bash[23952]: cluster 2024-07-28T19:29:30.948443+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-07-28T19:29:33.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:32 smithi100 bash[21828]: cluster 2024-07-28T19:29:31.952372+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-07-28T19:29:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:32 smithi100 bash[31715]: cluster 2024-07-28T19:29:31.952372+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-07-28T19:29:33.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:32 smithi138 bash[23952]: cluster 2024-07-28T19:29:31.952372+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-07-28T19:29:34.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:33 smithi100 bash[31715]: cluster 2024-07-28T19:29:32.875831+0000 mgr.y (mgr.14391) 130 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 0 B/s wr, 5 op/s 2024-07-28T19:29:34.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:33 smithi100 bash[21828]: cluster 2024-07-28T19:29:32.875831+0000 mgr.y (mgr.14391) 130 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 0 B/s wr, 5 op/s 2024-07-28T19:29:34.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:33 smithi138 bash[23952]: cluster 2024-07-28T19:29:32.875831+0000 mgr.y (mgr.14391) 130 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 0 B/s wr, 5 op/s 2024-07-28T19:29:34.923 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-07-28T19:29:34.932 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-07-28T19:29:34.932 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-07-28T19:29:34.932 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph mgr dump --format=json 2024-07-28T19:29:36.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:35 smithi100 bash[21828]: cluster 2024-07-28T19:29:34.876488+0000 mgr.y (mgr.14391) 131 : cluster [DBG] pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:36.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:35 smithi100 bash[31715]: cluster 2024-07-28T19:29:34.876488+0000 mgr.y (mgr.14391) 131 : cluster [DBG] pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:36.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:35 smithi138 bash[23952]: cluster 2024-07-28T19:29:34.876488+0000 mgr.y (mgr.14391) 131 : cluster [DBG] pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:36.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:29:36 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:29:38.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:37 smithi100 bash[21828]: cluster 2024-07-28T19:29:36.877543+0000 mgr.y (mgr.14391) 132 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:38.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:29:38 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:38] "GET /metrics HTTP/1.1" 200 239454 "" "Prometheus/2.43.0" 2024-07-28T19:29:38.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:37 smithi100 bash[31715]: cluster 2024-07-28T19:29:36.877543+0000 mgr.y (mgr.14391) 132 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:38.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:37 smithi138 bash[23952]: cluster 2024-07-28T19:29:36.877543+0000 mgr.y (mgr.14391) 132 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:39.707 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:29:40.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:39 smithi100 bash[21828]: cluster 2024-07-28T19:29:38.878650+0000 mgr.y (mgr.14391) 133 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:40.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:39 smithi100 bash[31715]: cluster 2024-07-28T19:29:38.878650+0000 mgr.y (mgr.14391) 133 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:40.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:39 smithi138 bash[23952]: cluster 2024-07-28T19:29:38.878650+0000 mgr.y (mgr.14391) 133 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:41 smithi100 bash[21828]: cluster 2024-07-28T19:29:40.879125+0000 mgr.y (mgr.14391) 134 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:42.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:41 smithi100 bash[31715]: cluster 2024-07-28T19:29:40.879125+0000 mgr.y (mgr.14391) 134 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:41 smithi138 bash[23952]: cluster 2024-07-28T19:29:40.879125+0000 mgr.y (mgr.14391) 134 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-07-28T19:29:42.871 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:29:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:42 smithi100 bash[21828]: audit 2024-07-28T19:29:42.867403+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.100:0/3180788185' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:29:43.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:42 smithi100 bash[31715]: audit 2024-07-28T19:29:42.867403+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.100:0/3180788185' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:29:43.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:42 smithi138 bash[23952]: audit 2024-07-28T19:29:42.867403+0000 mon.a (mon.0) 808 : audit [DBG] from='client.? 172.21.15.100:0/3180788185' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:29:43.540 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":20,"flags":0,"active_gid":14391,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6800","nonce":2345539946},{"type":"v1","addr":"172.21.15.100:6801","nonce":2345539946}]},"active_addr":"172.21.15.100:6801/2345539946","active_change":"2024-07-28T19:26:30.807599+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24280,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.100:8443/","prometheus":"http://172.21.15.100:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":3916188162}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":3784806487}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":376427751}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":480614525}]}]} 2024-07-28T19:29:43.545 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-07-28T19:29:43.545 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-07-28T19:29:43.545 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd dump --format=json 2024-07-28T19:29:44.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:43 smithi100 bash[21828]: cluster 2024-07-28T19:29:42.879751+0000 mgr.y (mgr.14391) 135 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:44.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:43 smithi100 bash[31715]: cluster 2024-07-28T19:29:42.879751+0000 mgr.y (mgr.14391) 135 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:44.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:43 smithi138 bash[23952]: cluster 2024-07-28T19:29:42.879751+0000 mgr.y (mgr.14391) 135 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:46.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:45 smithi100 bash[21828]: cluster 2024-07-28T19:29:44.880358+0000 mgr.y (mgr.14391) 136 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:46.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:46 smithi100 bash[21828]: audit 2024-07-28T19:29:45.908043+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:46.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:46 smithi100 bash[31715]: cluster 2024-07-28T19:29:44.880358+0000 mgr.y (mgr.14391) 136 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:46.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:46 smithi100 bash[31715]: audit 2024-07-28T19:29:45.908043+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:46.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:29:46 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:29:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:45 smithi138 bash[23952]: cluster 2024-07-28T19:29:44.880358+0000 mgr.y (mgr.14391) 136 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:46.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:45 smithi138 bash[23952]: audit 2024-07-28T19:29:45.908043+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:29:48.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:48 smithi138 bash[23952]: cluster 2024-07-28T19:29:46.881422+0000 mgr.y (mgr.14391) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:48.323 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:29:48.450 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:48 smithi100 bash[21828]: cluster 2024-07-28T19:29:46.881422+0000 mgr.y (mgr.14391) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:48.450 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:48 smithi100 bash[31715]: cluster 2024-07-28T19:29:46.881422+0000 mgr.y (mgr.14391) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:48.451 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:29:48 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:48] "GET /metrics HTTP/1.1" 200 239451 "" "Prometheus/2.43.0" 2024-07-28T19:29:50.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:50 smithi100 bash[31715]: cluster 2024-07-28T19:29:48.882067+0000 mgr.y (mgr.14391) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:50.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:50 smithi100 bash[21828]: cluster 2024-07-28T19:29:48.882067+0000 mgr.y (mgr.14391) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:50.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:50 smithi138 bash[23952]: cluster 2024-07-28T19:29:48.882067+0000 mgr.y (mgr.14391) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:51.608 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:29:51.608 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":72,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","created":"2024-07-28T19:16:00.842363+0000","modified":"2024-07-28T19:29:31.941843+0000","last_up_change":"2024-07-28T19:25:33.970932+0000","last_in_change":"2024-07-28T19:25:08.436673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-28T19:21:19.886730+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-28T19:27:14.397490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-07-28T19:29:14.173216+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-07-28T19:29:14.664032+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"c4a49140-8022-42e2-bba9-333eb0e88545","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6803","nonce":2528423215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6805","nonce":2528423215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6809","nonce":2528423215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6807","nonce":2528423215}]},"public_addr":"172.21.15.100:6803/2528423215","cluster_addr":"172.21.15.100:6805/2528423215","heartbeat_back_addr":"172.21.15.100:6809/2528423215","heartbeat_front_addr":"172.21.15.100:6807/2528423215","state":["exists","up"]},{"osd":1,"uuid":"4117fbb5-c88e-4d12-bc76-77050b2c0425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6811","nonce":3874116596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6813","nonce":3874116596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6817","nonce":3874116596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6815","nonce":3874116596}]},"public_addr":"172.21.15.100:6811/3874116596","cluster_addr":"172.21.15.100:6813/3874116596","heartbeat_back_addr":"172.21.15.100:6817/3874116596","heartbeat_front_addr":"172.21.15.100:6815/3874116596","state":["exists","up"]},{"osd":2,"uuid":"f3416396-83c7-42fb-9153-9cb806281347","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6819","nonce":2139838255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6821","nonce":2139838255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6825","nonce":2139838255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6823","nonce":2139838255}]},"public_addr":"172.21.15.100:6819/2139838255","cluster_addr":"172.21.15.100:6821/2139838255","heartbeat_back_addr":"172.21.15.100:6825/2139838255","heartbeat_front_addr":"172.21.15.100:6823/2139838255","state":["exists","up"]},{"osd":3,"uuid":"0363a205-8262-42a4-aaaf-e52558fec7f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6827","nonce":684533287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6829","nonce":684533287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6832","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6833","nonce":684533287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6831","nonce":684533287}]},"public_addr":"172.21.15.100:6827/684533287","cluster_addr":"172.21.15.100:6829/684533287","heartbeat_back_addr":"172.21.15.100:6833/684533287","heartbeat_front_addr":"172.21.15.100:6831/684533287","state":["exists","up"]},{"osd":4,"uuid":"75f51136-e1cc-4d3a-89d8-1618aaff366e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6801","nonce":1306402242}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6803","nonce":1306402242}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6807","nonce":1306402242}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6805","nonce":1306402242}]},"public_addr":"172.21.15.138:6801/1306402242","cluster_addr":"172.21.15.138:6803/1306402242","heartbeat_back_addr":"172.21.15.138:6807/1306402242","heartbeat_front_addr":"172.21.15.138:6805/1306402242","state":["exists","up"]},{"osd":5,"uuid":"0a3e4950-a201-4105-9e45-ef45ce2d5f3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6809","nonce":1102784487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6811","nonce":1102784487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6815","nonce":1102784487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6813","nonce":1102784487}]},"public_addr":"172.21.15.138:6809/1102784487","cluster_addr":"172.21.15.138:6811/1102784487","heartbeat_back_addr":"172.21.15.138:6815/1102784487","heartbeat_front_addr":"172.21.15.138:6813/1102784487","state":["exists","up"]},{"osd":6,"uuid":"8a20da48-aee9-44d8-a3db-63f916c6b763","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6817","nonce":3709444988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6819","nonce":3709444988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6823","nonce":3709444988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6821","nonce":3709444988}]},"public_addr":"172.21.15.138:6817/3709444988","cluster_addr":"172.21.15.138:6819/3709444988","heartbeat_back_addr":"172.21.15.138:6823/3709444988","heartbeat_front_addr":"172.21.15.138:6821/3709444988","state":["exists","up"]},{"osd":7,"uuid":"d52b6b70-462f-42ed-966f-fa14921ba172","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6825","nonce":1158882219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6827","nonce":1158882219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6831","nonce":1158882219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6829","nonce":1158882219}]},"public_addr":"172.21.15.138:6825/1158882219","cluster_addr":"172.21.15.138:6827/1158882219","heartbeat_back_addr":"172.21.15.138:6831/1158882219","heartbeat_front_addr":"172.21.15.138:6829/1158882219","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:19:30.916069+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:20:23.352940+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:21:16.853645+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:09.540873+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:59.198788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:23:49.135324+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:24:40.795346+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:25:32.389079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.100:6835/4040051651":"2024-07-29T19:29:22.449384+0000","172.21.15.100:0/638886582":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3154304581":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6801/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/2863303498":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/989116380":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3841450082":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6834/4040051651":"2024-07-29T19:29:22.449384+0000","172.21.15.100:0/553075364":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3152621533":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2291421462":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/1006451977":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3944711986":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6800/728100671":"2024-07-29T19:26:30.807202+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-28T19:29:52.230 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-07-28T19:29:52.231 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd dump --format=json 2024-07-28T19:29:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:52 smithi138 bash[23952]: cluster 2024-07-28T19:29:50.882664+0000 mgr.y (mgr.14391) 139 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:52 smithi138 bash[23952]: audit 2024-07-28T19:29:51.612542+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.100:0/1249840581' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:29:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:52 smithi100 bash[21828]: cluster 2024-07-28T19:29:50.882664+0000 mgr.y (mgr.14391) 139 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:52 smithi100 bash[21828]: audit 2024-07-28T19:29:51.612542+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.100:0/1249840581' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:29:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:52 smithi100 bash[31715]: cluster 2024-07-28T19:29:50.882664+0000 mgr.y (mgr.14391) 139 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:52 smithi100 bash[31715]: audit 2024-07-28T19:29:51.612542+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.100:0/1249840581' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:29:54.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:54 smithi138 bash[23952]: cluster 2024-07-28T19:29:52.883800+0000 mgr.y (mgr.14391) 140 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:54.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:54 smithi100 bash[21828]: cluster 2024-07-28T19:29:52.883800+0000 mgr.y (mgr.14391) 140 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:54.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:54 smithi100 bash[31715]: cluster 2024-07-28T19:29:52.883800+0000 mgr.y (mgr.14391) 140 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:56.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:29:56 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:29:56.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:56 smithi138 bash[23952]: cluster 2024-07-28T19:29:54.884512+0000 mgr.y (mgr.14391) 141 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:56.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:56 smithi100 bash[21828]: cluster 2024-07-28T19:29:54.884512+0000 mgr.y (mgr.14391) 141 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:56.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:56 smithi100 bash[31715]: cluster 2024-07-28T19:29:54.884512+0000 mgr.y (mgr.14391) 141 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:57.012 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:29:58.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:29:58 smithi138 bash[23952]: cluster 2024-07-28T19:29:56.885592+0000 mgr.y (mgr.14391) 142 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:58.311 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:29:58 smithi100 bash[21828]: cluster 2024-07-28T19:29:56.885592+0000 mgr.y (mgr.14391) 142 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:29:58.311 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:29:58 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:29:58] "GET /metrics HTTP/1.1" 200 239451 "" "Prometheus/2.43.0" 2024-07-28T19:29:58.311 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:29:58 smithi100 bash[31715]: cluster 2024-07-28T19:29:56.885592+0000 mgr.y (mgr.14391) 142 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:00.213 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:30:00.214 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":72,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","created":"2024-07-28T19:16:00.842363+0000","modified":"2024-07-28T19:29:31.941843+0000","last_up_change":"2024-07-28T19:25:33.970932+0000","last_in_change":"2024-07-28T19:25:08.436673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-28T19:21:19.886730+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-28T19:27:14.397490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-07-28T19:29:14.173216+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-07-28T19:29:14.664032+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"c4a49140-8022-42e2-bba9-333eb0e88545","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6803","nonce":2528423215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6805","nonce":2528423215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6809","nonce":2528423215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":2528423215},{"type":"v1","addr":"172.21.15.100:6807","nonce":2528423215}]},"public_addr":"172.21.15.100:6803/2528423215","cluster_addr":"172.21.15.100:6805/2528423215","heartbeat_back_addr":"172.21.15.100:6809/2528423215","heartbeat_front_addr":"172.21.15.100:6807/2528423215","state":["exists","up"]},{"osd":1,"uuid":"4117fbb5-c88e-4d12-bc76-77050b2c0425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6811","nonce":3874116596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6813","nonce":3874116596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6817","nonce":3874116596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":3874116596},{"type":"v1","addr":"172.21.15.100:6815","nonce":3874116596}]},"public_addr":"172.21.15.100:6811/3874116596","cluster_addr":"172.21.15.100:6813/3874116596","heartbeat_back_addr":"172.21.15.100:6817/3874116596","heartbeat_front_addr":"172.21.15.100:6815/3874116596","state":["exists","up"]},{"osd":2,"uuid":"f3416396-83c7-42fb-9153-9cb806281347","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6819","nonce":2139838255}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6821","nonce":2139838255}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6825","nonce":2139838255}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":2139838255},{"type":"v1","addr":"172.21.15.100:6823","nonce":2139838255}]},"public_addr":"172.21.15.100:6819/2139838255","cluster_addr":"172.21.15.100:6821/2139838255","heartbeat_back_addr":"172.21.15.100:6825/2139838255","heartbeat_front_addr":"172.21.15.100:6823/2139838255","state":["exists","up"]},{"osd":3,"uuid":"0363a205-8262-42a4-aaaf-e52558fec7f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6827","nonce":684533287}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6829","nonce":684533287}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6832","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6833","nonce":684533287}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":684533287},{"type":"v1","addr":"172.21.15.100:6831","nonce":684533287}]},"public_addr":"172.21.15.100:6827/684533287","cluster_addr":"172.21.15.100:6829/684533287","heartbeat_back_addr":"172.21.15.100:6833/684533287","heartbeat_front_addr":"172.21.15.100:6831/684533287","state":["exists","up"]},{"osd":4,"uuid":"75f51136-e1cc-4d3a-89d8-1618aaff366e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6801","nonce":1306402242}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6803","nonce":1306402242}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6807","nonce":1306402242}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":1306402242},{"type":"v1","addr":"172.21.15.138:6805","nonce":1306402242}]},"public_addr":"172.21.15.138:6801/1306402242","cluster_addr":"172.21.15.138:6803/1306402242","heartbeat_back_addr":"172.21.15.138:6807/1306402242","heartbeat_front_addr":"172.21.15.138:6805/1306402242","state":["exists","up"]},{"osd":5,"uuid":"0a3e4950-a201-4105-9e45-ef45ce2d5f3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6809","nonce":1102784487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6811","nonce":1102784487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6815","nonce":1102784487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6813","nonce":1102784487}]},"public_addr":"172.21.15.138:6809/1102784487","cluster_addr":"172.21.15.138:6811/1102784487","heartbeat_back_addr":"172.21.15.138:6815/1102784487","heartbeat_front_addr":"172.21.15.138:6813/1102784487","state":["exists","up"]},{"osd":6,"uuid":"8a20da48-aee9-44d8-a3db-63f916c6b763","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6817","nonce":3709444988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6819","nonce":3709444988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6823","nonce":3709444988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6821","nonce":3709444988}]},"public_addr":"172.21.15.138:6817/3709444988","cluster_addr":"172.21.15.138:6819/3709444988","heartbeat_back_addr":"172.21.15.138:6823/3709444988","heartbeat_front_addr":"172.21.15.138:6821/3709444988","state":["exists","up"]},{"osd":7,"uuid":"d52b6b70-462f-42ed-966f-fa14921ba172","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6825","nonce":1158882219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6827","nonce":1158882219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6831","nonce":1158882219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6829","nonce":1158882219}]},"public_addr":"172.21.15.138:6825/1158882219","cluster_addr":"172.21.15.138:6827/1158882219","heartbeat_back_addr":"172.21.15.138:6831/1158882219","heartbeat_front_addr":"172.21.15.138:6829/1158882219","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:19:30.916069+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:20:23.352940+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:21:16.853645+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:09.540873+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:59.198788+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:23:49.135324+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:24:40.795346+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:25:32.389079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.100:6835/4040051651":"2024-07-29T19:29:22.449384+0000","172.21.15.100:0/638886582":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3154304581":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6801/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/2863303498":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/989116380":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3841450082":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6834/4040051651":"2024-07-29T19:29:22.449384+0000","172.21.15.100:0/553075364":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:6800/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3152621533":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2291421462":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6801/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/1006451977":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3944711986":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6800/728100671":"2024-07-29T19:26:30.807202+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-28T19:30:00.225 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:00 smithi100 bash[31715]: cluster 2024-07-28T19:29:58.886212+0000 mgr.y (mgr.14391) 143 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:00.225 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:00 smithi100 bash[31715]: cluster 2024-07-28T19:30:00.000144+0000 mon.a (mon.0) 809 : cluster [INF] overall HEALTH_OK 2024-07-28T19:30:00.225 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:00 smithi100 bash[21828]: cluster 2024-07-28T19:29:58.886212+0000 mgr.y (mgr.14391) 143 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:00.225 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:00 smithi100 bash[21828]: cluster 2024-07-28T19:30:00.000144+0000 mon.a (mon.0) 809 : cluster [INF] overall HEALTH_OK 2024-07-28T19:30:00.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:00 smithi138 bash[23952]: cluster 2024-07-28T19:29:58.886212+0000 mgr.y (mgr.14391) 143 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:00.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:00 smithi138 bash[23952]: cluster 2024-07-28T19:30:00.000144+0000 mon.a (mon.0) 809 : cluster [INF] overall HEALTH_OK 2024-07-28T19:30:00.796 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-07-28T19:30:00.797 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-07-28T19:30:00.797 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-07-28T19:30:00.797 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-07-28T19:30:00.798 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-07-28T19:30:00.798 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-07-28T19:30:00.799 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-07-28T19:30:00.799 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-07-28T19:30:01.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:01 smithi100 bash[21828]: audit 2024-07-28T19:30:00.217769+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.100:0/3838304426' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:30:01.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:01 smithi100 bash[21828]: audit 2024-07-28T19:30:00.908402+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:01 smithi100 bash[31715]: audit 2024-07-28T19:30:00.217769+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.100:0/3838304426' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:30:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:01 smithi100 bash[31715]: audit 2024-07-28T19:30:00.908402+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:01.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:01 smithi138 bash[23952]: audit 2024-07-28T19:30:00.217769+0000 mon.a (mon.0) 810 : audit [DBG] from='client.? 172.21.15.100:0/3838304426' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:30:01.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:01 smithi138 bash[23952]: audit 2024-07-28T19:30:00.908402+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:02 smithi138 bash[23952]: cluster 2024-07-28T19:30:00.886675+0000 mgr.y (mgr.14391) 144 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:02.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:02 smithi100 bash[21828]: cluster 2024-07-28T19:30:00.886675+0000 mgr.y (mgr.14391) 144 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:02.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:02 smithi100 bash[31715]: cluster 2024-07-28T19:30:00.886675+0000 mgr.y (mgr.14391) 144 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:04.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:04 smithi138 bash[23952]: cluster 2024-07-28T19:30:02.887680+0000 mgr.y (mgr.14391) 145 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:04.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:04 smithi100 bash[21828]: cluster 2024-07-28T19:30:02.887680+0000 mgr.y (mgr.14391) 145 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:04.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:04 smithi100 bash[31715]: cluster 2024-07-28T19:30:02.887680+0000 mgr.y (mgr.14391) 145 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:05.718 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:05.718 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:05.719 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:05.720 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:05.721 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:05.722 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:05.722 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:05.724 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:06.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:06 smithi100 bash[21828]: cluster 2024-07-28T19:30:04.888272+0000 mgr.y (mgr.14391) 146 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:06.341 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:06 smithi100 bash[31715]: cluster 2024-07-28T19:30:04.888272+0000 mgr.y (mgr.14391) 146 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:06.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:30:06 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:30:06.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:06 smithi138 bash[23952]: cluster 2024-07-28T19:30:04.888272+0000 mgr.y (mgr.14391) 146 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:08.369 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:08 smithi100 bash[21828]: cluster 2024-07-28T19:30:06.889328+0000 mgr.y (mgr.14391) 147 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:08.369 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:30:08 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:08] "GET /metrics HTTP/1.1" 200 239427 "" "Prometheus/2.43.0" 2024-07-28T19:30:08.370 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:08 smithi100 bash[31715]: cluster 2024-07-28T19:30:06.889328+0000 mgr.y (mgr.14391) 147 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:08.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:08 smithi138 bash[23952]: cluster 2024-07-28T19:30:06.889328+0000 mgr.y (mgr.14391) 147 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:10.170 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:10 smithi100 bash[31715]: cluster 2024-07-28T19:30:08.889959+0000 mgr.y (mgr.14391) 148 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:10.171 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:10 smithi100 bash[21828]: cluster 2024-07-28T19:30:08.889959+0000 mgr.y (mgr.14391) 148 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:10.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:10 smithi138 bash[23952]: cluster 2024-07-28T19:30:08.889959+0000 mgr.y (mgr.14391) 148 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:12.403 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:12 smithi100 bash[31715]: cluster 2024-07-28T19:30:10.890547+0000 mgr.y (mgr.14391) 149 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:12.403 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:12 smithi100 bash[21828]: cluster 2024-07-28T19:30:10.890547+0000 mgr.y (mgr.14391) 149 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:12.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:12 smithi138 bash[23952]: cluster 2024-07-28T19:30:10.890547+0000 mgr.y (mgr.14391) 149 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:14.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:14 smithi100 bash[21828]: cluster 2024-07-28T19:30:12.891555+0000 mgr.y (mgr.14391) 150 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:14.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:14 smithi100 bash[31715]: cluster 2024-07-28T19:30:12.891555+0000 mgr.y (mgr.14391) 150 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:14.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:14 smithi138 bash[23952]: cluster 2024-07-28T19:30:12.891555+0000 mgr.y (mgr.14391) 150 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:16.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:16 smithi100 bash[21828]: cluster 2024-07-28T19:30:14.892187+0000 mgr.y (mgr.14391) 151 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:16.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:16 smithi100 bash[21828]: audit 2024-07-28T19:30:15.909401+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:16.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:16 smithi100 bash[31715]: cluster 2024-07-28T19:30:14.892187+0000 mgr.y (mgr.14391) 151 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:16.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:16 smithi100 bash[31715]: audit 2024-07-28T19:30:15.909401+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:16.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:30:16 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:30:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:16 smithi138 bash[23952]: cluster 2024-07-28T19:30:14.892187+0000 mgr.y (mgr.14391) 151 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:16 smithi138 bash[23952]: audit 2024-07-28T19:30:15.909401+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:18.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:18 smithi100 bash[21828]: cluster 2024-07-28T19:30:16.892850+0000 mgr.y (mgr.14391) 152 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:18.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:30:18 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:18] "GET /metrics HTTP/1.1" 200 239408 "" "Prometheus/2.43.0" 2024-07-28T19:30:18.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:18 smithi100 bash[31715]: cluster 2024-07-28T19:30:16.892850+0000 mgr.y (mgr.14391) 152 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:18.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:18 smithi138 bash[23952]: cluster 2024-07-28T19:30:16.892850+0000 mgr.y (mgr.14391) 152 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:19.249 INFO:teuthology.orchestra.run.smithi100.stdout:77309411439 2024-07-28T19:30:19.249 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-07-28T19:30:20.405 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:20 smithi100 bash[31715]: cluster 2024-07-28T19:30:18.893534+0000 mgr.y (mgr.14391) 153 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:20.406 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:20 smithi100 bash[21828]: cluster 2024-07-28T19:30:18.893534+0000 mgr.y (mgr.14391) 153 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:20.407 INFO:teuthology.orchestra.run.smithi100.stdout:111669149795 2024-07-28T19:30:20.407 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-07-28T19:30:20.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:20 smithi138 bash[23952]: cluster 2024-07-28T19:30:18.893534+0000 mgr.y (mgr.14391) 153 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:21.558 INFO:teuthology.orchestra.run.smithi100.stdout:55834574969 2024-07-28T19:30:21.559 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-07-28T19:30:21.993 INFO:teuthology.orchestra.run.smithi100.stdout:34359738499 2024-07-28T19:30:21.993 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-07-28T19:30:22.155 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:22 smithi100 bash[21828]: cluster 2024-07-28T19:30:20.894304+0000 mgr.y (mgr.14391) 154 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:22.155 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:22 smithi100 bash[31715]: cluster 2024-07-28T19:30:20.894304+0000 mgr.y (mgr.14391) 154 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:22.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:22 smithi138 bash[23952]: cluster 2024-07-28T19:30:20.894304+0000 mgr.y (mgr.14391) 154 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:23.887 INFO:teuthology.orchestra.run.smithi100.stdout:236223201341 2024-07-28T19:30:23.888 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-07-28T19:30:24.022 INFO:teuthology.orchestra.run.smithi100.stdout:133143986265 2024-07-28T19:30:24.023 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-07-28T19:30:24.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:24 smithi100 bash[21828]: cluster 2024-07-28T19:30:22.895116+0000 mgr.y (mgr.14391) 155 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:24.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:24 smithi100 bash[31715]: cluster 2024-07-28T19:30:22.895116+0000 mgr.y (mgr.14391) 155 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:24.431 INFO:teuthology.orchestra.run.smithi100.stdout:201863462983 2024-07-28T19:30:24.431 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-07-28T19:30:24.484 INFO:teuthology.orchestra.run.smithi100.stdout:167503724625 2024-07-28T19:30:24.484 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-07-28T19:30:24.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:24 smithi138 bash[23952]: cluster 2024-07-28T19:30:22.895116+0000 mgr.y (mgr.14391) 155 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:26.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:26 smithi100 bash[21828]: cluster 2024-07-28T19:30:24.895741+0000 mgr.y (mgr.14391) 156 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:26.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:26 smithi100 bash[31715]: cluster 2024-07-28T19:30:24.895741+0000 mgr.y (mgr.14391) 156 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:26 smithi138 bash[23952]: cluster 2024-07-28T19:30:24.895741+0000 mgr.y (mgr.14391) 156 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:26.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:30:26 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:30:27.076 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:27.077 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:27.078 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:27.078 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:27.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:27 smithi100 bash[21828]: audit 2024-07-28T19:30:26.885855+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:30:27.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:27 smithi100 bash[31715]: audit 2024-07-28T19:30:26.885855+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:30:27.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:27 smithi138 bash[23952]: audit 2024-07-28T19:30:26.885855+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:30:28.095 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:28.096 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:28.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:28] "GET /metrics HTTP/1.1" 200 239408 "" "Prometheus/2.43.0" 2024-07-28T19:30:28.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[21828]: cluster 2024-07-28T19:30:26.896754+0000 mgr.y (mgr.14391) 157 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:28.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[21828]: audit 2024-07-28T19:30:27.344046+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:30:28.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[21828]: audit 2024-07-28T19:30:27.345259+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:30:28.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[21828]: audit 2024-07-28T19:30:27.356896+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:30:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[31715]: cluster 2024-07-28T19:30:26.896754+0000 mgr.y (mgr.14391) 157 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[31715]: audit 2024-07-28T19:30:27.344046+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:30:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[31715]: audit 2024-07-28T19:30:27.345259+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:30:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:28 smithi100 bash[31715]: audit 2024-07-28T19:30:27.356896+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:30:28.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:28 smithi138 bash[23952]: cluster 2024-07-28T19:30:26.896754+0000 mgr.y (mgr.14391) 157 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:28.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:28 smithi138 bash[23952]: audit 2024-07-28T19:30:27.344046+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:30:28.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:28 smithi138 bash[23952]: audit 2024-07-28T19:30:27.345259+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:30:28.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:28 smithi138 bash[23952]: audit 2024-07-28T19:30:27.356896+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:30:29.122 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:29.123 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:30.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:30 smithi100 bash[21828]: cluster 2024-07-28T19:30:28.897510+0000 mgr.y (mgr.14391) 158 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:30.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:30 smithi100 bash[31715]: cluster 2024-07-28T19:30:28.897510+0000 mgr.y (mgr.14391) 158 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:30.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:30 smithi138 bash[23952]: cluster 2024-07-28T19:30:28.897510+0000 mgr.y (mgr.14391) 158 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:31.399 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:31 smithi100 bash[21828]: audit 2024-07-28T19:30:30.909154+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:31.399 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:31 smithi100 bash[31715]: audit 2024-07-28T19:30:30.909154+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:31.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:31 smithi138 bash[23952]: audit 2024-07-28T19:30:30.909154+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:32.239 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:32 smithi100 bash[21828]: cluster 2024-07-28T19:30:30.898225+0000 mgr.y (mgr.14391) 159 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:32.240 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:32 smithi100 bash[31715]: cluster 2024-07-28T19:30:30.898225+0000 mgr.y (mgr.14391) 159 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:32.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:32 smithi138 bash[23952]: cluster 2024-07-28T19:30:30.898225+0000 mgr.y (mgr.14391) 159 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:34.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:34 smithi100 bash[21828]: cluster 2024-07-28T19:30:32.899072+0000 mgr.y (mgr.14391) 160 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:34.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:34 smithi100 bash[31715]: cluster 2024-07-28T19:30:32.899072+0000 mgr.y (mgr.14391) 160 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:34.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:34 smithi138 bash[23952]: cluster 2024-07-28T19:30:32.899072+0000 mgr.y (mgr.14391) 160 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:36.317 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:36 smithi100 bash[21828]: cluster 2024-07-28T19:30:34.899712+0000 mgr.y (mgr.14391) 161 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:36.318 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:36 smithi100 bash[31715]: cluster 2024-07-28T19:30:34.899712+0000 mgr.y (mgr.14391) 161 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:36.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:30:36 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:30:36.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:36 smithi138 bash[23952]: cluster 2024-07-28T19:30:34.899712+0000 mgr.y (mgr.14391) 161 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:38.388 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:38 smithi100 bash[21828]: cluster 2024-07-28T19:30:36.900747+0000 mgr.y (mgr.14391) 162 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:38.388 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:38 smithi100 bash[31715]: cluster 2024-07-28T19:30:36.900747+0000 mgr.y (mgr.14391) 162 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:38.388 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:30:38 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:38] "GET /metrics HTTP/1.1" 200 239406 "" "Prometheus/2.43.0" 2024-07-28T19:30:38.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:38 smithi138 bash[23952]: cluster 2024-07-28T19:30:36.900747+0000 mgr.y (mgr.14391) 162 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:38.700 INFO:teuthology.orchestra.run.smithi100.stdout:55834574973 2024-07-28T19:30:39.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:39 smithi100 bash[21828]: audit 2024-07-28T19:30:38.706314+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.100:0/1807538174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:30:39.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:39 smithi100 bash[31715]: audit 2024-07-28T19:30:38.706314+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.100:0/1807538174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:30:39.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:39 smithi138 bash[23952]: audit 2024-07-28T19:30:38.706314+0000 mon.c (mon.2) 123 : audit [DBG] from='client.? 172.21.15.100:0/1807538174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:30:40.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:40 smithi100 bash[31715]: cluster 2024-07-28T19:30:38.901440+0000 mgr.y (mgr.14391) 163 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:40.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:40 smithi100 bash[21828]: cluster 2024-07-28T19:30:38.901440+0000 mgr.y (mgr.14391) 163 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:40.313 INFO:teuthology.orchestra.run.smithi100.stdout:77309411444 2024-07-28T19:30:40.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:40 smithi138 bash[23952]: cluster 2024-07-28T19:30:38.901440+0000 mgr.y (mgr.14391) 163 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:40.959 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574969 got 55834574973 for osd.1 2024-07-28T19:30:40.960 DEBUG:teuthology.parallel:result is None 2024-07-28T19:30:41.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:41 smithi100 bash[21828]: audit 2024-07-28T19:30:40.319565+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.100:0/3591000706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:30:41.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:41 smithi100 bash[31715]: audit 2024-07-28T19:30:40.319565+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.100:0/3591000706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:30:41.362 INFO:teuthology.orchestra.run.smithi100.stdout:111669149799 2024-07-28T19:30:41.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:41 smithi138 bash[23952]: audit 2024-07-28T19:30:40.319565+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.100:0/3591000706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:30:42.408 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:42 smithi100 bash[21828]: cluster 2024-07-28T19:30:40.901960+0000 mgr.y (mgr.14391) 164 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:42.409 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:42 smithi100 bash[21828]: audit 2024-07-28T19:30:41.368919+0000 mon.c (mon.2) 124 : audit [DBG] from='client.? 172.21.15.100:0/498208496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:30:42.409 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:42 smithi100 bash[31715]: cluster 2024-07-28T19:30:40.901960+0000 mgr.y (mgr.14391) 164 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:42.409 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:42 smithi100 bash[31715]: audit 2024-07-28T19:30:41.368919+0000 mon.c (mon.2) 124 : audit [DBG] from='client.? 172.21.15.100:0/498208496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:30:42.449 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411439 got 77309411444 for osd.2 2024-07-28T19:30:42.449 DEBUG:teuthology.parallel:result is None 2024-07-28T19:30:42.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:42 smithi138 bash[23952]: cluster 2024-07-28T19:30:40.901960+0000 mgr.y (mgr.14391) 164 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:42.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:42 smithi138 bash[23952]: audit 2024-07-28T19:30:41.368919+0000 mon.c (mon.2) 124 : audit [DBG] from='client.? 172.21.15.100:0/498208496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:30:42.689 INFO:teuthology.orchestra.run.smithi100.stdout:133143986269 2024-07-28T19:30:42.961 INFO:teuthology.orchestra.run.smithi100.stdout:236223201344 2024-07-28T19:30:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:43 smithi100 bash[21828]: audit 2024-07-28T19:30:42.696205+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.100:0/2006506588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:30:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:43 smithi100 bash[21828]: audit 2024-07-28T19:30:42.967284+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.100:0/2248195706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:30:43.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:43 smithi100 bash[31715]: audit 2024-07-28T19:30:42.696205+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.100:0/2006506588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:30:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:43 smithi100 bash[31715]: audit 2024-07-28T19:30:42.967284+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.100:0/2248195706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:30:43.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:43 smithi138 bash[23952]: audit 2024-07-28T19:30:42.696205+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.100:0/2006506588' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:30:43.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:43 smithi138 bash[23952]: audit 2024-07-28T19:30:42.967284+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.100:0/2248195706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:30:43.971 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149795 got 111669149799 for osd.3 2024-07-28T19:30:43.971 DEBUG:teuthology.parallel:result is None 2024-07-28T19:30:44.037 INFO:teuthology.orchestra.run.smithi100.stdout:167503724629 2024-07-28T19:30:44.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:44 smithi100 bash[31715]: cluster 2024-07-28T19:30:42.902970+0000 mgr.y (mgr.14391) 165 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:44.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:44 smithi100 bash[31715]: audit 2024-07-28T19:30:44.043645+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.100:0/3327940096' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:30:44.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:44 smithi100 bash[21828]: cluster 2024-07-28T19:30:42.902970+0000 mgr.y (mgr.14391) 165 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:44.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:44 smithi100 bash[21828]: audit 2024-07-28T19:30:44.043645+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.100:0/3327940096' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:30:44.381 INFO:teuthology.orchestra.run.smithi100.stdout:201863462987 2024-07-28T19:30:44.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:44 smithi138 bash[23952]: cluster 2024-07-28T19:30:42.902970+0000 mgr.y (mgr.14391) 165 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:44.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:44 smithi138 bash[23952]: audit 2024-07-28T19:30:44.043645+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.100:0/3327940096' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:30:45.180 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986265 got 133143986269 for osd.4 2024-07-28T19:30:45.180 DEBUG:teuthology.parallel:result is None 2024-07-28T19:30:45.272 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201341 got 236223201344 for osd.7 2024-07-28T19:30:45.273 INFO:teuthology.orchestra.run.smithi100.stdout:34359738504 2024-07-28T19:30:45.273 DEBUG:teuthology.parallel:result is None 2024-07-28T19:30:45.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:45 smithi100 bash[21828]: audit 2024-07-28T19:30:44.386925+0000 mon.c (mon.2) 125 : audit [DBG] from='client.? 172.21.15.100:0/2194603811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:30:45.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:45 smithi100 bash[31715]: audit 2024-07-28T19:30:44.386925+0000 mon.c (mon.2) 125 : audit [DBG] from='client.? 172.21.15.100:0/2194603811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:30:45.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:45 smithi138 bash[23952]: audit 2024-07-28T19:30:44.386925+0000 mon.c (mon.2) 125 : audit [DBG] from='client.? 172.21.15.100:0/2194603811' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:30:46.234 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:46 smithi100 bash[21828]: cluster 2024-07-28T19:30:44.903556+0000 mgr.y (mgr.14391) 166 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:46.234 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:46 smithi100 bash[21828]: audit 2024-07-28T19:30:45.278962+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.100:0/1462012780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:30:46.234 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:46 smithi100 bash[21828]: audit 2024-07-28T19:30:45.909847+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:46.234 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:46 smithi100 bash[31715]: cluster 2024-07-28T19:30:44.903556+0000 mgr.y (mgr.14391) 166 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:46.235 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:46 smithi100 bash[31715]: audit 2024-07-28T19:30:45.278962+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.100:0/1462012780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:30:46.235 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:46 smithi100 bash[31715]: audit 2024-07-28T19:30:45.909847+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:46.502 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724625 got 167503724629 for osd.5 2024-07-28T19:30:46.502 DEBUG:teuthology.parallel:result is None 2024-07-28T19:30:46.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:30:46 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:30:46.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:46 smithi138 bash[23952]: cluster 2024-07-28T19:30:44.903556+0000 mgr.y (mgr.14391) 166 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:46.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:46 smithi138 bash[23952]: audit 2024-07-28T19:30:45.278962+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.100:0/1462012780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:30:46.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:46 smithi138 bash[23952]: audit 2024-07-28T19:30:45.909847+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:30:46.652 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462983 got 201863462987 for osd.6 2024-07-28T19:30:46.652 DEBUG:teuthology.parallel:result is None 2024-07-28T19:30:46.815 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738499 got 34359738504 for osd.0 2024-07-28T19:30:46.815 DEBUG:teuthology.parallel:result is None 2024-07-28T19:30:46.815 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-07-28T19:30:46.816 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph pg dump --format=json 2024-07-28T19:30:48.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:48 smithi100 bash[21828]: cluster 2024-07-28T19:30:46.904603+0000 mgr.y (mgr.14391) 167 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:48.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:30:48 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:48] "GET /metrics HTTP/1.1" 200 239409 "" "Prometheus/2.43.0" 2024-07-28T19:30:48.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:48 smithi100 bash[31715]: cluster 2024-07-28T19:30:46.904603+0000 mgr.y (mgr.14391) 167 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:48.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:48 smithi138 bash[23952]: cluster 2024-07-28T19:30:46.904603+0000 mgr.y (mgr.14391) 167 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:50.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:50 smithi100 bash[21828]: cluster 2024-07-28T19:30:48.905224+0000 mgr.y (mgr.14391) 168 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:50.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:50 smithi100 bash[31715]: cluster 2024-07-28T19:30:48.905224+0000 mgr.y (mgr.14391) 168 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:50.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:50 smithi138 bash[23952]: cluster 2024-07-28T19:30:48.905224+0000 mgr.y (mgr.14391) 168 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:51.612 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:30:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:52 smithi100 bash[21828]: cluster 2024-07-28T19:30:50.905849+0000 mgr.y (mgr.14391) 169 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:52 smithi100 bash[31715]: cluster 2024-07-28T19:30:50.905849+0000 mgr.y (mgr.14391) 169 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:52.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:52 smithi138 bash[23952]: cluster 2024-07-28T19:30:50.905849+0000 mgr.y (mgr.14391) 169 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:54.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:54 smithi100 bash[21828]: cluster 2024-07-28T19:30:52.906504+0000 mgr.y (mgr.14391) 170 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:54.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:54 smithi100 bash[31715]: cluster 2024-07-28T19:30:52.906504+0000 mgr.y (mgr.14391) 170 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:54.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:54 smithi138 bash[23952]: cluster 2024-07-28T19:30:52.906504+0000 mgr.y (mgr.14391) 170 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:54.820 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:30:54.824 INFO:teuthology.orchestra.run.smithi100.stderr:dumped all 2024-07-28T19:30:55.500 INFO:teuthology.orchestra.run.smithi100.stdout:{"pg_ready":true,"pg_map":{"version":146,"stamp":"2024-07-28T19:30:52.906107+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":259,"num_read_kb":355,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386420,"kb_used_data":7576,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574796,"statfs":{"total":767960285184,"available":765516591104,"internally_reserved":0,"allocated":7757824,"data_stored":4366413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004452"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-07-28T19:29:32.082110+0000","last_change":"2024-07-28T19:29:32.082110+0000","last_active":"2024-07-28T19:29:32.082110+0000","last_peered":"2024-07-28T19:29:32.082110+0000","last_clean":"2024-07-28T19:29:32.082110+0000","last_became_active":"2024-07-28T19:29:32.081261+0000","last_became_peered":"2024-07-28T19:29:32.081261+0000","last_unstale":"2024-07-28T19:29:32.082110+0000","last_undegraded":"2024-07-28T19:29:32.082110+0000","last_fullsized":"2024-07-28T19:29:32.082110+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:05:16.608598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.946643+0000","last_change":"2024-07-28T19:29:15.300221+0000","last_active":"2024-07-28T19:29:30.946643+0000","last_peered":"2024-07-28T19:29:30.946643+0000","last_clean":"2024-07-28T19:29:30.946643+0000","last_became_active":"2024-07-28T19:29:15.298656+0000","last_became_peered":"2024-07-28T19:29:15.298656+0000","last_unstale":"2024-07-28T19:29:30.946643+0000","last_undegraded":"2024-07-28T19:29:30.946643+0000","last_fullsized":"2024-07-28T19:29:30.946643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:00:31.428501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.950975+0000","last_change":"2024-07-28T19:29:16.447233+0000","last_active":"2024-07-28T19:29:30.950975+0000","last_peered":"2024-07-28T19:29:30.950975+0000","last_clean":"2024-07-28T19:29:30.950975+0000","last_became_active":"2024-07-28T19:29:16.446518+0000","last_became_peered":"2024-07-28T19:29:16.446518+0000","last_unstale":"2024-07-28T19:29:30.950975+0000","last_undegraded":"2024-07-28T19:29:30.950975+0000","last_fullsized":"2024-07-28T19:29:30.950975+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:34:35.744242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.948944+0000","last_change":"2024-07-28T19:29:15.283186+0000","last_active":"2024-07-28T19:29:30.948944+0000","last_peered":"2024-07-28T19:29:30.948944+0000","last_clean":"2024-07-28T19:29:30.948944+0000","last_became_active":"2024-07-28T19:29:15.282685+0000","last_became_peered":"2024-07-28T19:29:15.282685+0000","last_unstale":"2024-07-28T19:29:30.948944+0000","last_undegraded":"2024-07-28T19:29:30.948944+0000","last_fullsized":"2024-07-28T19:29:30.948944+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:06:18.152760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.948973+0000","last_change":"2024-07-28T19:29:16.450398+0000","last_active":"2024-07-28T19:29:30.948973+0000","last_peered":"2024-07-28T19:29:30.948973+0000","last_clean":"2024-07-28T19:29:30.948973+0000","last_became_active":"2024-07-28T19:29:16.449594+0000","last_became_peered":"2024-07-28T19:29:16.449594+0000","last_unstale":"2024-07-28T19:29:30.948973+0000","last_undegraded":"2024-07-28T19:29:30.948973+0000","last_fullsized":"2024-07-28T19:29:30.948973+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:28:35.449786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.946639+0000","last_change":"2024-07-28T19:29:15.299547+0000","last_active":"2024-07-28T19:29:30.946639+0000","last_peered":"2024-07-28T19:29:30.946639+0000","last_clean":"2024-07-28T19:29:30.946639+0000","last_became_active":"2024-07-28T19:29:15.296937+0000","last_became_peered":"2024-07-28T19:29:15.296937+0000","last_unstale":"2024-07-28T19:29:30.946639+0000","last_undegraded":"2024-07-28T19:29:30.946639+0000","last_fullsized":"2024-07-28T19:29:30.946639+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:03:30.991662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958179+0000","last_change":"2024-07-28T19:29:16.448600+0000","last_active":"2024-07-28T19:29:30.958179+0000","last_peered":"2024-07-28T19:29:30.958179+0000","last_clean":"2024-07-28T19:29:30.958179+0000","last_became_active":"2024-07-28T19:29:16.447856+0000","last_became_peered":"2024-07-28T19:29:16.447856+0000","last_unstale":"2024-07-28T19:29:30.958179+0000","last_undegraded":"2024-07-28T19:29:30.958179+0000","last_fullsized":"2024-07-28T19:29:30.958179+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:18:48.990747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":27,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.600532+0000","last_change":"2024-07-28T19:29:15.264709+0000","last_active":"2024-07-28T19:29:31.600532+0000","last_peered":"2024-07-28T19:29:31.600532+0000","last_clean":"2024-07-28T19:29:31.600532+0000","last_became_active":"2024-07-28T19:29:15.261437+0000","last_became_peered":"2024-07-28T19:29:15.261437+0000","last_unstale":"2024-07-28T19:29:31.600532+0000","last_undegraded":"2024-07-28T19:29:31.600532+0000","last_fullsized":"2024-07-28T19:29:31.600532+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:22:34.843162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947782+0000","last_change":"2024-07-28T19:29:15.299396+0000","last_active":"2024-07-28T19:29:30.947782+0000","last_peered":"2024-07-28T19:29:30.947782+0000","last_clean":"2024-07-28T19:29:30.947782+0000","last_became_active":"2024-07-28T19:29:15.297605+0000","last_became_peered":"2024-07-28T19:29:15.297605+0000","last_unstale":"2024-07-28T19:29:30.947782+0000","last_undegraded":"2024-07-28T19:29:30.947782+0000","last_fullsized":"2024-07-28T19:29:30.947782+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:32:23.182916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.950776+0000","last_change":"2024-07-28T19:29:16.445468+0000","last_active":"2024-07-28T19:29:30.950776+0000","last_peered":"2024-07-28T19:29:30.950776+0000","last_clean":"2024-07-28T19:29:30.950776+0000","last_became_active":"2024-07-28T19:29:16.444693+0000","last_became_peered":"2024-07-28T19:29:16.444693+0000","last_unstale":"2024-07-28T19:29:30.950776+0000","last_undegraded":"2024-07-28T19:29:30.950776+0000","last_fullsized":"2024-07-28T19:29:30.950776+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:13:39.927285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947561+0000","last_change":"2024-07-28T19:29:15.300758+0000","last_active":"2024-07-28T19:29:30.947561+0000","last_peered":"2024-07-28T19:29:30.947561+0000","last_clean":"2024-07-28T19:29:30.947561+0000","last_became_active":"2024-07-28T19:29:15.299775+0000","last_became_peered":"2024-07-28T19:29:15.299775+0000","last_unstale":"2024-07-28T19:29:30.947561+0000","last_undegraded":"2024-07-28T19:29:30.947561+0000","last_fullsized":"2024-07-28T19:29:30.947561+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:50:19.096556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957388+0000","last_change":"2024-07-28T19:29:15.299875+0000","last_active":"2024-07-28T19:29:30.957388+0000","last_peered":"2024-07-28T19:29:30.957388+0000","last_clean":"2024-07-28T19:29:30.957388+0000","last_became_active":"2024-07-28T19:29:15.299085+0000","last_became_peered":"2024-07-28T19:29:15.299085+0000","last_unstale":"2024-07-28T19:29:30.957388+0000","last_undegraded":"2024-07-28T19:29:30.957388+0000","last_fullsized":"2024-07-28T19:29:30.957388+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:05:00.231080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947481+0000","last_change":"2024-07-28T19:29:15.279653+0000","last_active":"2024-07-28T19:29:30.947481+0000","last_peered":"2024-07-28T19:29:30.947481+0000","last_clean":"2024-07-28T19:29:30.947481+0000","last_became_active":"2024-07-28T19:29:15.278829+0000","last_became_peered":"2024-07-28T19:29:15.278829+0000","last_unstale":"2024-07-28T19:29:30.947481+0000","last_undegraded":"2024-07-28T19:29:30.947481+0000","last_fullsized":"2024-07-28T19:29:30.947481+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:46:07.127995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.602044+0000","last_change":"2024-07-28T19:29:16.445265+0000","last_active":"2024-07-28T19:29:31.602044+0000","last_peered":"2024-07-28T19:29:31.602044+0000","last_clean":"2024-07-28T19:29:31.602044+0000","last_became_active":"2024-07-28T19:29:16.444411+0000","last_became_peered":"2024-07-28T19:29:16.444411+0000","last_unstale":"2024-07-28T19:29:31.602044+0000","last_undegraded":"2024-07-28T19:29:31.602044+0000","last_fullsized":"2024-07-28T19:29:31.602044+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:32:45.312229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.951048+0000","last_change":"2024-07-28T19:29:15.267503+0000","last_active":"2024-07-28T19:29:30.951048+0000","last_peered":"2024-07-28T19:29:30.951048+0000","last_clean":"2024-07-28T19:29:30.951048+0000","last_became_active":"2024-07-28T19:29:15.266632+0000","last_became_peered":"2024-07-28T19:29:15.266632+0000","last_unstale":"2024-07-28T19:29:30.951048+0000","last_undegraded":"2024-07-28T19:29:30.951048+0000","last_fullsized":"2024-07-28T19:29:30.951048+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:41:34.344331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949118+0000","last_change":"2024-07-28T19:29:16.325716+0000","last_active":"2024-07-28T19:29:30.949118+0000","last_peered":"2024-07-28T19:29:30.949118+0000","last_clean":"2024-07-28T19:29:30.949118+0000","last_became_active":"2024-07-28T19:29:16.325194+0000","last_became_peered":"2024-07-28T19:29:16.325194+0000","last_unstale":"2024-07-28T19:29:30.949118+0000","last_undegraded":"2024-07-28T19:29:30.949118+0000","last_fullsized":"2024-07-28T19:29:30.949118+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:27:16.454707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.956318+0000","last_change":"2024-07-28T19:29:15.304133+0000","last_active":"2024-07-28T19:29:30.956318+0000","last_peered":"2024-07-28T19:29:30.956318+0000","last_clean":"2024-07-28T19:29:30.956318+0000","last_became_active":"2024-07-28T19:29:15.303800+0000","last_became_peered":"2024-07-28T19:29:15.303800+0000","last_unstale":"2024-07-28T19:29:30.956318+0000","last_undegraded":"2024-07-28T19:29:30.956318+0000","last_fullsized":"2024-07-28T19:29:30.956318+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:44:18.395540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957700+0000","last_change":"2024-07-28T19:29:15.307023+0000","last_active":"2024-07-28T19:29:30.957700+0000","last_peered":"2024-07-28T19:29:30.957700+0000","last_clean":"2024-07-28T19:29:30.957700+0000","last_became_active":"2024-07-28T19:29:15.306567+0000","last_became_peered":"2024-07-28T19:29:15.306567+0000","last_unstale":"2024-07-28T19:29:30.957700+0000","last_undegraded":"2024-07-28T19:29:30.957700+0000","last_fullsized":"2024-07-28T19:29:30.957700+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:06:31.551224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.600619+0000","last_change":"2024-07-28T19:29:15.280359+0000","last_active":"2024-07-28T19:29:31.600619+0000","last_peered":"2024-07-28T19:29:31.600619+0000","last_clean":"2024-07-28T19:29:31.600619+0000","last_became_active":"2024-07-28T19:29:15.279580+0000","last_became_peered":"2024-07-28T19:29:15.279580+0000","last_unstale":"2024-07-28T19:29:31.600619+0000","last_undegraded":"2024-07-28T19:29:31.600619+0000","last_fullsized":"2024-07-28T19:29:31.600619+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:07:48.900003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957978+0000","last_change":"2024-07-28T19:29:15.291033+0000","last_active":"2024-07-28T19:29:30.957978+0000","last_peered":"2024-07-28T19:29:30.957978+0000","last_clean":"2024-07-28T19:29:30.957978+0000","last_became_active":"2024-07-28T19:29:15.290414+0000","last_became_peered":"2024-07-28T19:29:15.290414+0000","last_unstale":"2024-07-28T19:29:30.957978+0000","last_undegraded":"2024-07-28T19:29:30.957978+0000","last_fullsized":"2024-07-28T19:29:30.957978+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:34:33.307390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957455+0000","last_change":"2024-07-28T19:29:15.259681+0000","last_active":"2024-07-28T19:29:30.957455+0000","last_peered":"2024-07-28T19:29:30.957455+0000","last_clean":"2024-07-28T19:29:30.957455+0000","last_became_active":"2024-07-28T19:29:15.258936+0000","last_became_peered":"2024-07-28T19:29:15.258936+0000","last_unstale":"2024-07-28T19:29:30.957455+0000","last_undegraded":"2024-07-28T19:29:30.957455+0000","last_fullsized":"2024-07-28T19:29:30.957455+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:47:44.039720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958162+0000","last_change":"2024-07-28T19:29:15.304430+0000","last_active":"2024-07-28T19:29:30.958162+0000","last_peered":"2024-07-28T19:29:30.958162+0000","last_clean":"2024-07-28T19:29:30.958162+0000","last_became_active":"2024-07-28T19:29:15.304053+0000","last_became_peered":"2024-07-28T19:29:15.304053+0000","last_unstale":"2024-07-28T19:29:30.958162+0000","last_undegraded":"2024-07-28T19:29:30.958162+0000","last_fullsized":"2024-07-28T19:29:30.958162+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:55:25.885784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955802+0000","last_change":"2024-07-28T19:29:15.280675+0000","last_active":"2024-07-28T19:29:30.955802+0000","last_peered":"2024-07-28T19:29:30.955802+0000","last_clean":"2024-07-28T19:29:30.955802+0000","last_became_active":"2024-07-28T19:29:15.280312+0000","last_became_peered":"2024-07-28T19:29:15.280312+0000","last_unstale":"2024-07-28T19:29:30.955802+0000","last_undegraded":"2024-07-28T19:29:30.955802+0000","last_fullsized":"2024-07-28T19:29:30.955802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:22:09.192184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.959576+0000","last_change":"2024-07-28T19:29:15.298989+0000","last_active":"2024-07-28T19:29:30.959576+0000","last_peered":"2024-07-28T19:29:30.959576+0000","last_clean":"2024-07-28T19:29:30.959576+0000","last_became_active":"2024-07-28T19:29:15.298703+0000","last_became_peered":"2024-07-28T19:29:15.298703+0000","last_unstale":"2024-07-28T19:29:30.959576+0000","last_undegraded":"2024-07-28T19:29:30.959576+0000","last_fullsized":"2024-07-28T19:29:30.959576+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:20:28.312665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":278,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.946869+0000","last_change":"2024-07-28T19:25:38.028024+0000","last_active":"2024-07-28T19:29:30.946869+0000","last_peered":"2024-07-28T19:29:30.946869+0000","last_clean":"2024-07-28T19:29:30.946869+0000","last_became_active":"2024-07-28T19:25:38.027110+0000","last_became_peered":"2024-07-28T19:25:38.027110+0000","last_unstale":"2024-07-28T19:29:30.946869+0000","last_undegraded":"2024-07-28T19:29:30.946869+0000","last_fullsized":"2024-07-28T19:29:30.946869+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_clean_scrub_stamp":"2024-07-28T19:21:20.215133+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:20:35.538874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.951021+0000","last_change":"2024-07-28T19:27:15.550979+0000","last_active":"2024-07-28T19:29:30.951021+0000","last_peered":"2024-07-28T19:29:30.951021+0000","last_clean":"2024-07-28T19:29:30.951021+0000","last_became_active":"2024-07-28T19:27:15.550214+0000","last_became_peered":"2024-07-28T19:27:15.550214+0000","last_unstale":"2024-07-28T19:29:30.951021+0000","last_undegraded":"2024-07-28T19:29:30.951021+0000","last_fullsized":"2024-07-28T19:29:30.951021+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:17:06.298582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955877+0000","last_change":"2024-07-28T19:29:15.262671+0000","last_active":"2024-07-28T19:29:30.955877+0000","last_peered":"2024-07-28T19:29:30.955877+0000","last_clean":"2024-07-28T19:29:30.955877+0000","last_became_active":"2024-07-28T19:29:15.262218+0000","last_became_peered":"2024-07-28T19:29:15.262218+0000","last_unstale":"2024-07-28T19:29:30.955877+0000","last_undegraded":"2024-07-28T19:29:30.955877+0000","last_fullsized":"2024-07-28T19:29:30.955877+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:41:19.915904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.601607+0000","last_change":"2024-07-28T19:29:16.446131+0000","last_active":"2024-07-28T19:29:31.601607+0000","last_peered":"2024-07-28T19:29:31.601607+0000","last_clean":"2024-07-28T19:29:31.601607+0000","last_became_active":"2024-07-28T19:29:16.445372+0000","last_became_peered":"2024-07-28T19:29:16.445372+0000","last_unstale":"2024-07-28T19:29:31.601607+0000","last_undegraded":"2024-07-28T19:29:31.601607+0000","last_fullsized":"2024-07-28T19:29:31.601607+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:38:05.866961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.946641+0000","last_change":"2024-07-28T19:27:15.702756+0000","last_active":"2024-07-28T19:29:30.946641+0000","last_peered":"2024-07-28T19:29:30.946641+0000","last_clean":"2024-07-28T19:29:30.946641+0000","last_became_active":"2024-07-28T19:27:15.701931+0000","last_became_peered":"2024-07-28T19:27:15.701931+0000","last_unstale":"2024-07-28T19:29:30.946641+0000","last_undegraded":"2024-07-28T19:29:30.946641+0000","last_fullsized":"2024-07-28T19:29:30.946641+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:20:20.149727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.948369+0000","last_change":"2024-07-28T19:29:15.281513+0000","last_active":"2024-07-28T19:29:30.948369+0000","last_peered":"2024-07-28T19:29:30.948369+0000","last_clean":"2024-07-28T19:29:30.948369+0000","last_became_active":"2024-07-28T19:29:15.280706+0000","last_became_peered":"2024-07-28T19:29:15.280706+0000","last_unstale":"2024-07-28T19:29:30.948369+0000","last_undegraded":"2024-07-28T19:29:30.948369+0000","last_fullsized":"2024-07-28T19:29:30.948369+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:07:49.929016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-07-28T19:29:32.517058+0000","last_change":"2024-07-28T19:29:32.517058+0000","last_active":"2024-07-28T19:29:32.517058+0000","last_peered":"2024-07-28T19:29:32.517058+0000","last_clean":"2024-07-28T19:29:32.517058+0000","last_became_active":"2024-07-28T19:29:32.516140+0000","last_became_peered":"2024-07-28T19:29:32.516140+0000","last_unstale":"2024-07-28T19:29:32.517058+0000","last_undegraded":"2024-07-28T19:29:32.517058+0000","last_fullsized":"2024-07-28T19:29:32.517058+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:54:48.068694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955310+0000","last_change":"2024-07-28T19:27:15.706055+0000","last_active":"2024-07-28T19:29:30.955310+0000","last_peered":"2024-07-28T19:29:30.955310+0000","last_clean":"2024-07-28T19:29:30.955310+0000","last_became_active":"2024-07-28T19:27:15.705102+0000","last_became_peered":"2024-07-28T19:27:15.705102+0000","last_unstale":"2024-07-28T19:29:30.955310+0000","last_undegraded":"2024-07-28T19:29:30.955310+0000","last_fullsized":"2024-07-28T19:29:30.955310+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:00:20.179809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.953902+0000","last_change":"2024-07-28T19:29:15.259870+0000","last_active":"2024-07-28T19:29:30.953902+0000","last_peered":"2024-07-28T19:29:30.953902+0000","last_clean":"2024-07-28T19:29:30.953902+0000","last_became_active":"2024-07-28T19:29:15.259169+0000","last_became_peered":"2024-07-28T19:29:15.259169+0000","last_unstale":"2024-07-28T19:29:30.953902+0000","last_undegraded":"2024-07-28T19:29:30.953902+0000","last_fullsized":"2024-07-28T19:29:30.953902+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:53:53.614089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.964769+0000","last_change":"2024-07-28T19:29:15.603688+0000","last_active":"2024-07-28T19:29:30.964769+0000","last_peered":"2024-07-28T19:29:30.964769+0000","last_clean":"2024-07-28T19:29:30.964769+0000","last_became_active":"2024-07-28T19:29:15.603137+0000","last_became_peered":"2024-07-28T19:29:15.603137+0000","last_unstale":"2024-07-28T19:29:30.964769+0000","last_undegraded":"2024-07-28T19:29:30.964769+0000","last_fullsized":"2024-07-28T19:29:30.964769+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:40:41.988368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.951086+0000","last_change":"2024-07-28T19:27:15.552985+0000","last_active":"2024-07-28T19:29:30.951086+0000","last_peered":"2024-07-28T19:29:30.951086+0000","last_clean":"2024-07-28T19:29:30.951086+0000","last_became_active":"2024-07-28T19:27:15.552104+0000","last_became_peered":"2024-07-28T19:27:15.552104+0000","last_unstale":"2024-07-28T19:29:30.951086+0000","last_undegraded":"2024-07-28T19:29:30.951086+0000","last_fullsized":"2024-07-28T19:29:30.951086+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:46:22.374425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957804+0000","last_change":"2024-07-28T19:29:15.278523+0000","last_active":"2024-07-28T19:29:30.957804+0000","last_peered":"2024-07-28T19:29:30.957804+0000","last_clean":"2024-07-28T19:29:30.957804+0000","last_became_active":"2024-07-28T19:29:15.277560+0000","last_became_peered":"2024-07-28T19:29:15.277560+0000","last_unstale":"2024-07-28T19:29:30.957804+0000","last_undegraded":"2024-07-28T19:29:30.957804+0000","last_fullsized":"2024-07-28T19:29:30.957804+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T06:26:39.820338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.960654+0000","last_change":"2024-07-28T19:29:15.297935+0000","last_active":"2024-07-28T19:29:30.960654+0000","last_peered":"2024-07-28T19:29:30.960654+0000","last_clean":"2024-07-28T19:29:30.960654+0000","last_became_active":"2024-07-28T19:29:15.297583+0000","last_became_peered":"2024-07-28T19:29:15.297583+0000","last_unstale":"2024-07-28T19:29:30.960654+0000","last_undegraded":"2024-07-28T19:29:30.960654+0000","last_fullsized":"2024-07-28T19:29:30.960654+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:47:11.793168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.952020+0000","last_change":"2024-07-28T19:29:15.266697+0000","last_active":"2024-07-28T19:29:30.952020+0000","last_peered":"2024-07-28T19:29:30.952020+0000","last_clean":"2024-07-28T19:29:30.952020+0000","last_became_active":"2024-07-28T19:29:15.265912+0000","last_became_peered":"2024-07-28T19:29:15.265912+0000","last_unstale":"2024-07-28T19:29:30.952020+0000","last_undegraded":"2024-07-28T19:29:30.952020+0000","last_fullsized":"2024-07-28T19:29:30.952020+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:03:55.516994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.960929+0000","last_change":"2024-07-28T19:27:15.706361+0000","last_active":"2024-07-28T19:29:30.960929+0000","last_peered":"2024-07-28T19:29:30.960929+0000","last_clean":"2024-07-28T19:29:30.960929+0000","last_became_active":"2024-07-28T19:27:15.705499+0000","last_became_peered":"2024-07-28T19:27:15.705499+0000","last_unstale":"2024-07-28T19:29:30.960929+0000","last_undegraded":"2024-07-28T19:29:30.960929+0000","last_fullsized":"2024-07-28T19:29:30.960929+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:11:20.803138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-07-28T19:29:32.516389+0000","last_change":"2024-07-28T19:29:32.516389+0000","last_active":"2024-07-28T19:29:32.516389+0000","last_peered":"2024-07-28T19:29:32.516389+0000","last_clean":"2024-07-28T19:29:32.516389+0000","last_became_active":"2024-07-28T19:29:32.515454+0000","last_became_peered":"2024-07-28T19:29:32.515454+0000","last_unstale":"2024-07-28T19:29:32.516389+0000","last_undegraded":"2024-07-28T19:29:32.516389+0000","last_fullsized":"2024-07-28T19:29:32.516389+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:44:35.842708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.960797+0000","last_change":"2024-07-28T19:29:15.267962+0000","last_active":"2024-07-28T19:29:30.960797+0000","last_peered":"2024-07-28T19:29:30.960797+0000","last_clean":"2024-07-28T19:29:30.960797+0000","last_became_active":"2024-07-28T19:29:15.267350+0000","last_became_peered":"2024-07-28T19:29:15.267350+0000","last_unstale":"2024-07-28T19:29:30.960797+0000","last_undegraded":"2024-07-28T19:29:30.960797+0000","last_fullsized":"2024-07-28T19:29:30.960797+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:14:25.642811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947464+0000","last_change":"2024-07-28T19:27:15.703022+0000","last_active":"2024-07-28T19:29:30.947464+0000","last_peered":"2024-07-28T19:29:30.947464+0000","last_clean":"2024-07-28T19:29:30.947464+0000","last_became_active":"2024-07-28T19:27:15.701965+0000","last_became_peered":"2024-07-28T19:27:15.701965+0000","last_unstale":"2024-07-28T19:29:30.947464+0000","last_undegraded":"2024-07-28T19:29:30.947464+0000","last_fullsized":"2024-07-28T19:29:30.947464+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:18:56.026871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949263+0000","last_change":"2024-07-28T19:29:16.448067+0000","last_active":"2024-07-28T19:29:30.949263+0000","last_peered":"2024-07-28T19:29:30.949263+0000","last_clean":"2024-07-28T19:29:30.949263+0000","last_became_active":"2024-07-28T19:29:16.447172+0000","last_became_peered":"2024-07-28T19:29:16.447172+0000","last_unstale":"2024-07-28T19:29:30.949263+0000","last_undegraded":"2024-07-28T19:29:30.949263+0000","last_fullsized":"2024-07-28T19:29:30.949263+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:53:50.969459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958440+0000","last_change":"2024-07-28T19:29:15.281381+0000","last_active":"2024-07-28T19:29:30.958440+0000","last_peered":"2024-07-28T19:29:30.958440+0000","last_clean":"2024-07-28T19:29:30.958440+0000","last_became_active":"2024-07-28T19:29:15.280586+0000","last_became_peered":"2024-07-28T19:29:15.280586+0000","last_unstale":"2024-07-28T19:29:30.958440+0000","last_undegraded":"2024-07-28T19:29:30.958440+0000","last_fullsized":"2024-07-28T19:29:30.958440+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:17:11.213449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.961792+0000","last_change":"2024-07-28T19:27:15.704938+0000","last_active":"2024-07-28T19:29:30.961792+0000","last_peered":"2024-07-28T19:29:30.961792+0000","last_clean":"2024-07-28T19:29:30.961792+0000","last_became_active":"2024-07-28T19:27:15.703726+0000","last_became_peered":"2024-07-28T19:27:15.703726+0000","last_unstale":"2024-07-28T19:29:30.961792+0000","last_undegraded":"2024-07-28T19:29:30.961792+0000","last_fullsized":"2024-07-28T19:29:30.961792+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:39:26.136768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955363+0000","last_change":"2024-07-28T19:29:16.454445+0000","last_active":"2024-07-28T19:29:30.955363+0000","last_peered":"2024-07-28T19:29:30.955363+0000","last_clean":"2024-07-28T19:29:30.955363+0000","last_became_active":"2024-07-28T19:29:16.453274+0000","last_became_peered":"2024-07-28T19:29:16.453274+0000","last_unstale":"2024-07-28T19:29:30.955363+0000","last_undegraded":"2024-07-28T19:29:30.955363+0000","last_fullsized":"2024-07-28T19:29:30.955363+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:43:40.314510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949287+0000","last_change":"2024-07-28T19:29:15.278341+0000","last_active":"2024-07-28T19:29:30.949287+0000","last_peered":"2024-07-28T19:29:30.949287+0000","last_clean":"2024-07-28T19:29:30.949287+0000","last_became_active":"2024-07-28T19:29:15.277543+0000","last_became_peered":"2024-07-28T19:29:15.277543+0000","last_unstale":"2024-07-28T19:29:30.949287+0000","last_undegraded":"2024-07-28T19:29:30.949287+0000","last_fullsized":"2024-07-28T19:29:30.949287+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:31:05.176791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.600441+0000","last_change":"2024-07-28T19:27:15.560013+0000","last_active":"2024-07-28T19:29:31.600441+0000","last_peered":"2024-07-28T19:29:31.600441+0000","last_clean":"2024-07-28T19:29:31.600441+0000","last_became_active":"2024-07-28T19:27:15.559241+0000","last_became_peered":"2024-07-28T19:27:15.559241+0000","last_unstale":"2024-07-28T19:29:31.600441+0000","last_undegraded":"2024-07-28T19:29:31.600441+0000","last_fullsized":"2024-07-28T19:29:31.600441+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:41:17.422708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957169+0000","last_change":"2024-07-28T19:29:15.303436+0000","last_active":"2024-07-28T19:29:30.957169+0000","last_peered":"2024-07-28T19:29:30.957169+0000","last_clean":"2024-07-28T19:29:30.957169+0000","last_became_active":"2024-07-28T19:29:15.302771+0000","last_became_peered":"2024-07-28T19:29:15.302771+0000","last_unstale":"2024-07-28T19:29:30.957169+0000","last_undegraded":"2024-07-28T19:29:30.957169+0000","last_fullsized":"2024-07-28T19:29:30.957169+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:27:55.834358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957585+0000","last_change":"2024-07-28T19:29:15.303958+0000","last_active":"2024-07-28T19:29:30.957585+0000","last_peered":"2024-07-28T19:29:30.957585+0000","last_clean":"2024-07-28T19:29:30.957585+0000","last_became_active":"2024-07-28T19:29:15.303509+0000","last_became_peered":"2024-07-28T19:29:15.303509+0000","last_unstale":"2024-07-28T19:29:30.957585+0000","last_undegraded":"2024-07-28T19:29:30.957585+0000","last_fullsized":"2024-07-28T19:29:30.957585+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:18:55.497305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"70'7","reported_seq":27,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957823+0000","last_change":"2024-07-28T19:29:15.281929+0000","last_active":"2024-07-28T19:29:30.957823+0000","last_peered":"2024-07-28T19:29:30.957823+0000","last_clean":"2024-07-28T19:29:30.957823+0000","last_became_active":"2024-07-28T19:29:15.277637+0000","last_became_peered":"2024-07-28T19:29:15.277637+0000","last_unstale":"2024-07-28T19:29:30.957823+0000","last_undegraded":"2024-07-28T19:29:30.957823+0000","last_fullsized":"2024-07-28T19:29:30.957823+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:04:03.599599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.959794+0000","last_change":"2024-07-28T19:29:15.296240+0000","last_active":"2024-07-28T19:29:30.959794+0000","last_peered":"2024-07-28T19:29:30.959794+0000","last_clean":"2024-07-28T19:29:30.959794+0000","last_became_active":"2024-07-28T19:29:15.295869+0000","last_became_peered":"2024-07-28T19:29:15.295869+0000","last_unstale":"2024-07-28T19:29:30.959794+0000","last_undegraded":"2024-07-28T19:29:30.959794+0000","last_fullsized":"2024-07-28T19:29:30.959794+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:12:38.976248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947195+0000","last_change":"2024-07-28T19:29:15.299299+0000","last_active":"2024-07-28T19:29:30.947195+0000","last_peered":"2024-07-28T19:29:30.947195+0000","last_clean":"2024-07-28T19:29:30.947195+0000","last_became_active":"2024-07-28T19:29:15.296865+0000","last_became_peered":"2024-07-28T19:29:15.296865+0000","last_unstale":"2024-07-28T19:29:30.947195+0000","last_undegraded":"2024-07-28T19:29:30.947195+0000","last_fullsized":"2024-07-28T19:29:30.947195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:54:46.661711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.952007+0000","last_change":"2024-07-28T19:29:15.277680+0000","last_active":"2024-07-28T19:29:30.952007+0000","last_peered":"2024-07-28T19:29:30.952007+0000","last_clean":"2024-07-28T19:29:30.952007+0000","last_became_active":"2024-07-28T19:29:15.276939+0000","last_became_peered":"2024-07-28T19:29:15.276939+0000","last_unstale":"2024-07-28T19:29:30.952007+0000","last_undegraded":"2024-07-28T19:29:30.952007+0000","last_fullsized":"2024-07-28T19:29:30.952007+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:39:43.008228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955925+0000","last_change":"2024-07-28T19:29:16.447973+0000","last_active":"2024-07-28T19:29:30.955925+0000","last_peered":"2024-07-28T19:29:30.955925+0000","last_clean":"2024-07-28T19:29:30.955925+0000","last_became_active":"2024-07-28T19:29:16.447208+0000","last_became_peered":"2024-07-28T19:29:16.447208+0000","last_unstale":"2024-07-28T19:29:30.955925+0000","last_undegraded":"2024-07-28T19:29:30.955925+0000","last_fullsized":"2024-07-28T19:29:30.955925+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:16:12.017505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949716+0000","last_change":"2024-07-28T19:29:15.284313+0000","last_active":"2024-07-28T19:29:30.949716+0000","last_peered":"2024-07-28T19:29:30.949716+0000","last_clean":"2024-07-28T19:29:30.949716+0000","last_became_active":"2024-07-28T19:29:15.283919+0000","last_became_peered":"2024-07-28T19:29:15.283919+0000","last_unstale":"2024-07-28T19:29:30.949716+0000","last_undegraded":"2024-07-28T19:29:30.949716+0000","last_fullsized":"2024-07-28T19:29:30.949716+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:18:48.974717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955152+0000","last_change":"2024-07-28T19:29:16.293216+0000","last_active":"2024-07-28T19:29:30.955152+0000","last_peered":"2024-07-28T19:29:30.955152+0000","last_clean":"2024-07-28T19:29:30.955152+0000","last_became_active":"2024-07-28T19:29:16.292635+0000","last_became_peered":"2024-07-28T19:29:16.292635+0000","last_unstale":"2024-07-28T19:29:30.955152+0000","last_undegraded":"2024-07-28T19:29:30.955152+0000","last_fullsized":"2024-07-28T19:29:30.955152+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:09:58.615631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.601878+0000","last_change":"2024-07-28T19:29:16.276582+0000","last_active":"2024-07-28T19:29:31.601878+0000","last_peered":"2024-07-28T19:29:31.601878+0000","last_clean":"2024-07-28T19:29:31.601878+0000","last_became_active":"2024-07-28T19:29:16.275714+0000","last_became_peered":"2024-07-28T19:29:16.275714+0000","last_unstale":"2024-07-28T19:29:31.601878+0000","last_undegraded":"2024-07-28T19:29:31.601878+0000","last_fullsized":"2024-07-28T19:29:31.601878+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:49:26.417025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955571+0000","last_change":"2024-07-28T19:29:15.270203+0000","last_active":"2024-07-28T19:29:30.955571+0000","last_peered":"2024-07-28T19:29:30.955571+0000","last_clean":"2024-07-28T19:29:30.955571+0000","last_became_active":"2024-07-28T19:29:15.269343+0000","last_became_peered":"2024-07-28T19:29:15.269343+0000","last_unstale":"2024-07-28T19:29:30.955571+0000","last_undegraded":"2024-07-28T19:29:30.955571+0000","last_fullsized":"2024-07-28T19:29:30.955571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:01:57.007719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.948765+0000","last_change":"2024-07-28T19:29:16.446263+0000","last_active":"2024-07-28T19:29:30.948765+0000","last_peered":"2024-07-28T19:29:30.948765+0000","last_clean":"2024-07-28T19:29:30.948765+0000","last_became_active":"2024-07-28T19:29:16.445988+0000","last_became_peered":"2024-07-28T19:29:16.445988+0000","last_unstale":"2024-07-28T19:29:30.948765+0000","last_undegraded":"2024-07-28T19:29:30.948765+0000","last_fullsized":"2024-07-28T19:29:30.948765+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:52:15.881155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.963237+0000","last_change":"2024-07-28T19:29:15.270401+0000","last_active":"2024-07-28T19:29:30.963237+0000","last_peered":"2024-07-28T19:29:30.963237+0000","last_clean":"2024-07-28T19:29:30.963237+0000","last_became_active":"2024-07-28T19:29:15.268811+0000","last_became_peered":"2024-07-28T19:29:15.268811+0000","last_unstale":"2024-07-28T19:29:30.963237+0000","last_undegraded":"2024-07-28T19:29:30.963237+0000","last_fullsized":"2024-07-28T19:29:30.963237+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:45:42.090959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957923+0000","last_change":"2024-07-28T19:29:16.327119+0000","last_active":"2024-07-28T19:29:30.957923+0000","last_peered":"2024-07-28T19:29:30.957923+0000","last_clean":"2024-07-28T19:29:30.957923+0000","last_became_active":"2024-07-28T19:29:16.326690+0000","last_became_peered":"2024-07-28T19:29:16.326690+0000","last_unstale":"2024-07-28T19:29:30.957923+0000","last_undegraded":"2024-07-28T19:29:30.957923+0000","last_fullsized":"2024-07-28T19:29:30.957923+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:00:14.430162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958582+0000","last_change":"2024-07-28T19:29:15.248377+0000","last_active":"2024-07-28T19:29:30.958582+0000","last_peered":"2024-07-28T19:29:30.958582+0000","last_clean":"2024-07-28T19:29:30.958582+0000","last_became_active":"2024-07-28T19:29:15.247531+0000","last_became_peered":"2024-07-28T19:29:15.247531+0000","last_unstale":"2024-07-28T19:29:30.958582+0000","last_undegraded":"2024-07-28T19:29:30.958582+0000","last_fullsized":"2024-07-28T19:29:30.958582+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:24:12.524223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.956601+0000","last_change":"2024-07-28T19:29:15.297883+0000","last_active":"2024-07-28T19:29:30.956601+0000","last_peered":"2024-07-28T19:29:30.956601+0000","last_clean":"2024-07-28T19:29:30.956601+0000","last_became_active":"2024-07-28T19:29:15.297582+0000","last_became_peered":"2024-07-28T19:29:15.297582+0000","last_unstale":"2024-07-28T19:29:30.956601+0000","last_undegraded":"2024-07-28T19:29:30.956601+0000","last_fullsized":"2024-07-28T19:29:30.956601+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T06:18:22.320889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949495+0000","last_change":"2024-07-28T19:29:15.279264+0000","last_active":"2024-07-28T19:29:30.949495+0000","last_peered":"2024-07-28T19:29:30.949495+0000","last_clean":"2024-07-28T19:29:30.949495+0000","last_became_active":"2024-07-28T19:29:15.278728+0000","last_became_peered":"2024-07-28T19:29:15.278728+0000","last_unstale":"2024-07-28T19:29:30.949495+0000","last_undegraded":"2024-07-28T19:29:30.949495+0000","last_fullsized":"2024-07-28T19:29:30.949495+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:28:34.240386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957527+0000","last_change":"2024-07-28T19:29:15.307357+0000","last_active":"2024-07-28T19:29:30.957527+0000","last_peered":"2024-07-28T19:29:30.957527+0000","last_clean":"2024-07-28T19:29:30.957527+0000","last_became_active":"2024-07-28T19:29:15.306819+0000","last_became_peered":"2024-07-28T19:29:15.306819+0000","last_unstale":"2024-07-28T19:29:30.957527+0000","last_undegraded":"2024-07-28T19:29:30.957527+0000","last_fullsized":"2024-07-28T19:29:30.957527+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:50:24.186555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.951068+0000","last_change":"2024-07-28T19:29:15.302679+0000","last_active":"2024-07-28T19:29:30.951068+0000","last_peered":"2024-07-28T19:29:30.951068+0000","last_clean":"2024-07-28T19:29:30.951068+0000","last_became_active":"2024-07-28T19:29:15.302156+0000","last_became_peered":"2024-07-28T19:29:15.302156+0000","last_unstale":"2024-07-28T19:29:30.951068+0000","last_undegraded":"2024-07-28T19:29:30.951068+0000","last_fullsized":"2024-07-28T19:29:30.951068+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:54:06.863534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958487+0000","last_change":"2024-07-28T19:29:15.600588+0000","last_active":"2024-07-28T19:29:30.958487+0000","last_peered":"2024-07-28T19:29:30.958487+0000","last_clean":"2024-07-28T19:29:30.958487+0000","last_became_active":"2024-07-28T19:29:15.599767+0000","last_became_peered":"2024-07-28T19:29:15.599767+0000","last_unstale":"2024-07-28T19:29:30.958487+0000","last_undegraded":"2024-07-28T19:29:30.958487+0000","last_fullsized":"2024-07-28T19:29:30.958487+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:09:05.954358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.952021+0000","last_change":"2024-07-28T19:29:15.292421+0000","last_active":"2024-07-28T19:29:30.952021+0000","last_peered":"2024-07-28T19:29:30.952021+0000","last_clean":"2024-07-28T19:29:30.952021+0000","last_became_active":"2024-07-28T19:29:15.291035+0000","last_became_peered":"2024-07-28T19:29:15.291035+0000","last_unstale":"2024-07-28T19:29:30.952021+0000","last_undegraded":"2024-07-28T19:29:30.952021+0000","last_fullsized":"2024-07-28T19:29:30.952021+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:51:06.527335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.956219+0000","last_change":"2024-07-28T19:29:16.446102+0000","last_active":"2024-07-28T19:29:30.956219+0000","last_peered":"2024-07-28T19:29:30.956219+0000","last_clean":"2024-07-28T19:29:30.956219+0000","last_became_active":"2024-07-28T19:29:16.445138+0000","last_became_peered":"2024-07-28T19:29:16.445138+0000","last_unstale":"2024-07-28T19:29:30.956219+0000","last_undegraded":"2024-07-28T19:29:30.956219+0000","last_fullsized":"2024-07-28T19:29:30.956219+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:35:18.586253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957629+0000","last_change":"2024-07-28T19:29:15.281836+0000","last_active":"2024-07-28T19:29:30.957629+0000","last_peered":"2024-07-28T19:29:30.957629+0000","last_clean":"2024-07-28T19:29:30.957629+0000","last_became_active":"2024-07-28T19:29:15.281461+0000","last_became_peered":"2024-07-28T19:29:15.281461+0000","last_unstale":"2024-07-28T19:29:30.957629+0000","last_undegraded":"2024-07-28T19:29:30.957629+0000","last_fullsized":"2024-07-28T19:29:30.957629+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:57:35.879475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.956901+0000","last_change":"2024-07-28T19:29:15.304058+0000","last_active":"2024-07-28T19:29:30.956901+0000","last_peered":"2024-07-28T19:29:30.956901+0000","last_clean":"2024-07-28T19:29:30.956901+0000","last_became_active":"2024-07-28T19:29:15.303594+0000","last_became_peered":"2024-07-28T19:29:15.303594+0000","last_unstale":"2024-07-28T19:29:30.956901+0000","last_undegraded":"2024-07-28T19:29:30.956901+0000","last_fullsized":"2024-07-28T19:29:30.956901+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:22:07.859902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":22,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949793+0000","last_change":"2024-07-28T19:29:15.270210+0000","last_active":"2024-07-28T19:29:30.949793+0000","last_peered":"2024-07-28T19:29:30.949793+0000","last_clean":"2024-07-28T19:29:30.949793+0000","last_became_active":"2024-07-28T19:29:15.268610+0000","last_became_peered":"2024-07-28T19:29:15.268610+0000","last_unstale":"2024-07-28T19:29:30.949793+0000","last_undegraded":"2024-07-28T19:29:30.949793+0000","last_fullsized":"2024-07-28T19:29:30.949793+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:07:50.326353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201347,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298680,"kb_used_data":1324,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446472,"statfs":{"total":95995035648,"available":95689187328,"internally_reserved":0,"allocated":1355776,"data_stored":915530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.85299999999999998,"15min":0.86899999999999999},"min":{"1min":0.52300000000000002,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.181,"5min":1.228,"15min":1.228},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.91000000000000003,"15min":0.91900000000000004},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":1.0289999999999999}]},{"osd":1,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.85299999999999998,"15min":0.85499999999999998},"min":{"1min":0.67000000000000004,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2110000000000001,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.93400000000000005,"15min":0.92800000000000005},"min":{"1min":0.70599999999999996,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.81599999999999995}]},{"osd":2,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.97199999999999998,"15min":0.97799999999999998},"min":{"1min":0.62,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.163,"5min":1.284,"15min":1.284},"last":1.085},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.97299999999999998,"15min":1.0009999999999999},"min":{"1min":0.63300000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2629999999999999,"5min":1.296,"15min":1.296},"last":1.3560000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.64100000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.1619999999999999,"5min":1.345,"15min":1.345},"last":0.96599999999999997},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.94999999999999996,"15min":0.96399999999999997},"min":{"1min":0.78300000000000003,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":1.117}]},{"osd":4,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.88900000000000001,"15min":0.88300000000000001},"min":{"1min":0.623,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.151},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95599999999999996,"15min":1},"min":{"1min":0.60399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.2549999999999999,"5min":1.474,"15min":1.474},"last":1.171}]},{"osd":5,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.93200000000000005,"15min":0.95999999999999996},"min":{"1min":0.65100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.2170000000000001,"5min":1.272,"15min":1.272},"last":1.3},{"interface":"front","average":{"1min":1.02,"5min":1.026,"15min":1.038},"min":{"1min":0.69399999999999995,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.331,"5min":1.335,"15min":1.335},"last":1.242}]},{"osd":6,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0669999999999999,"15min":1.085},"min":{"1min":0.751,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.377,"5min":1.385,"15min":1.385},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.0609999999999999,"15min":1.093},"min":{"1min":0.746,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.4930000000000001,"5min":1.581,"15min":1.581},"last":1.268}]}]},{"osd":6,"up_from":47,"seq":201863462989,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":915442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:29:50 2024","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.83499999999999996,"15min":0.83599999999999997},"min":{"1min":0.623,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.90300000000000002,"15min":0.89300000000000002},"min":{"1min":0.56699999999999995,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.198,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.0189999999999999}]},{"osd":1,"last update":"Sun Jul 28 19:29:50 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.88900000000000001,"15min":0.85999999999999999},"min":{"1min":0.54300000000000004,"5min":0.503,"15min":0.503},"max":{"1min":3.2679999999999998,"5min":3.2679999999999998,"15min":3.2679999999999998},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.90000000000000002,"15min":0.86399999999999999},"min":{"1min":0.72499999999999998,"5min":0.499,"15min":0.499},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.89700000000000002}]},{"osd":2,"last update":"Sun Jul 28 19:29:50 2024","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.90400000000000003,"15min":0.86499999999999999},"min":{"1min":0.71499999999999997,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.1379999999999999,"5min":1.222,"15min":1.222},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.93799999999999994,"15min":0.90400000000000003},"min":{"1min":0.70799999999999996,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3080000000000001,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":1.0620000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:29:50 2024","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":0.96699999999999997,"15min":0.93500000000000005},"min":{"1min":0.60199999999999998,"5min":0.498,"15min":0.498},"max":{"1min":3.492,"5min":3.492,"15min":3.492},"last":0.88},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.93799999999999994,"15min":0.878},"min":{"1min":0.73799999999999999,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":0.995}]},{"osd":4,"last update":"Sun Jul 28 19:29:50 2024","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.86699999999999999,"15min":0.82199999999999995},"min":{"1min":0.63100000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.9380000000000002,"5min":2.9380000000000002,"15min":2.9380000000000002},"last":0.75},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.96799999999999997,"15min":0.84999999999999998},"min":{"1min":0.73399999999999999,"5min":0.316,"15min":0.316},"max":{"1min":3.3919999999999999,"5min":3.3919999999999999,"15min":3.3919999999999999},"last":0.629}]},{"osd":5,"last update":"Sun Jul 28 19:29:50 2024","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.94099999999999995,"15min":0.84099999999999997},"min":{"1min":0.75700000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.399,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.167,"5min":0.98999999999999999,"15min":0.85899999999999999},"min":{"1min":0.68300000000000005,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.665,"5min":3.665,"15min":3.665},"last":0.79100000000000004}]},{"osd":7,"last update":"Sun Jul 28 19:30:38 2024","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.073,"15min":1.0629999999999999},"min":{"1min":0.78100000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":1.228,"5min":1.1160000000000001,"15min":1.101},"min":{"1min":0.77700000000000002,"5min":0.379,"15min":0.379},"max":{"1min":3.5790000000000002,"5min":3.5790000000000002,"15min":3.5790000000000002},"last":0.94399999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724631,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298080,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447072,"statfs":{"total":95995035648,"available":95689801728,"internally_reserved":0,"allocated":741376,"data_stored":324034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.91000000000000003,"15min":0.86799999999999999},"min":{"1min":0.58099999999999996,"5min":0.55000000000000004,"15min":0.50900000000000001},"max":{"1min":1.1990000000000001,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":1.0089999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.874,"15min":0.84799999999999998},"min":{"1min":0.73099999999999998,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1100000000000001,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":1.0640000000000001}]},{"osd":1,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.91700000000000004,"15min":0.873},"min":{"1min":0.59799999999999998,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.2769999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":1.1439999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.90300000000000002,"15min":0.86899999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.077,"5min":1.347,"15min":1.347},"last":0.81499999999999995}]},{"osd":2,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.89800000000000002,"15min":0.871},"min":{"1min":0.55300000000000005,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.1779999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.93500000000000005,"15min":0.89700000000000002},"min":{"1min":0.60699999999999998,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.345,"5min":1.345,"15min":1.345},"last":0.92500000000000004}]},{"osd":3,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.93100000000000005,"15min":0.89700000000000002},"min":{"1min":0.66200000000000003,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1240000000000001,"5min":1.397,"15min":1.397},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.97499999999999998,"15min":0.89900000000000002},"min":{"1min":0.71299999999999997,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.1830000000000001,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":0.96099999999999997}]},{"osd":4,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.83699999999999997,"15min":0.74399999999999999},"min":{"1min":0.63200000000000001,"5min":0.39900000000000002,"15min":0.377},"max":{"1min":0.98899999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.86099999999999999,"15min":0.79800000000000004},"min":{"1min":0.64900000000000002,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":1.339,"5min":1.4870000000000001,"15min":1.4870000000000001},"last":1.1000000000000001}]},{"osd":6,"last update":"Sun Jul 28 19:29:53 2024","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.94199999999999995,"15min":0.86699999999999999},"min":{"1min":0.72799999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.1639999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.96899999999999997,"15min":0.92000000000000004},"min":{"1min":0.623,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.3680000000000001,"5min":2.1669999999999998,"15min":2.1669999999999998},"last":1.226}]},{"osd":7,"last update":"Sun Jul 28 19:30:46 2024","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.006,"15min":1},"min":{"1min":0.66200000000000003,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.2150000000000001,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.01,"15min":1.032},"min":{"1min":0.80100000000000005,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.343,"5min":1.516,"15min":1.516},"last":1.1819999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986272,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298092,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447060,"statfs":{"total":95995035648,"available":95689789440,"internally_reserved":0,"allocated":757760,"data_stored":325150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.88600000000000001,"15min":0.85399999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.85099999999999998,"15min":0.83199999999999996},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.2889999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":1.0249999999999999}]},{"osd":1,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.83799999999999997,"15min":0.80400000000000005},"min":{"1min":0.55200000000000005,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89900000000000002,"15min":0.872},"min":{"1min":0.60299999999999998,"5min":0.57399999999999995,"15min":0.55600000000000005},"max":{"1min":1.145,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.90900000000000003}]},{"osd":2,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.91200000000000003,"15min":0.89300000000000002},"min":{"1min":0.67700000000000005,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.1930000000000001,"5min":1.298,"15min":1.298},"last":1.1479999999999999},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.92000000000000004,"15min":0.875},"min":{"1min":0.56100000000000005,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2250000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.76100000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.94399999999999995,"15min":0.88800000000000001},"min":{"1min":0.78900000000000003,"5min":0.504,"15min":0.504},"max":{"1min":1.462,"5min":1.462,"15min":1.462},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.90000000000000002,"15min":0.86899999999999999},"min":{"1min":0.63100000000000001,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.2090000000000001,"5min":1.583,"15min":1.583},"last":0.97799999999999998}]},{"osd":5,"last update":"Sun Jul 28 19:30:07 2024","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.93200000000000005,"15min":0.82599999999999996},"min":{"1min":0.58099999999999996,"5min":0.40200000000000002,"15min":0.39000000000000001},"max":{"1min":1.244,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.92500000000000004,"15min":0.83899999999999997},"min":{"1min":0.59399999999999997,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":1.325,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":1.0529999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:29:50 2024","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.94299999999999995,"15min":0.91500000000000004},"min":{"1min":0.56599999999999995,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.3919999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.93400000000000005,"15min":0.86199999999999999},"min":{"1min":0.48599999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.1659999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.86299999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:30:39 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.117,"15min":1.3580000000000001},"min":{"1min":0.77800000000000002,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.4239999999999999,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.198,"15min":1.49},"min":{"1min":0.69299999999999995,"5min":0.378,"15min":0.378},"max":{"1min":1.423,"5min":10.920999999999999,"15min":10.920999999999999},"last":1.0980000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149802,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":741376,"data_stored":323807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.69299999999999995,"15min":0.68500000000000005},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.84099999999999997,"5min":1.909,"15min":1.909},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.79100000000000004,"15min":0.77400000000000002},"min":{"1min":0.36099999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.95599999999999996,"5min":1.556,"15min":1.556},"last":0.77600000000000002}]},{"osd":1,"last update":"Sun Jul 28 19:30:24 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.71499999999999997,"15min":0.70499999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.93400000000000005,"5min":1.583,"15min":1.583},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.69099999999999995,"15min":0.67300000000000004},"min":{"1min":0.39500000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.997,"5min":1.427,"15min":1.427},"last":0.89700000000000002}]},{"osd":2,"last update":"Sun Jul 28 19:30:24 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.79800000000000004,"15min":0.76400000000000001},"min":{"1min":0.45100000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.161,"5min":1.839,"15min":1.839},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.72299999999999998,"15min":0.67900000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.171,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.60399999999999998}]},{"osd":4,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.90900000000000003,"15min":0.88},"min":{"1min":0.68600000000000005,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.1080000000000001,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.97099999999999997,"15min":1.052},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.1619999999999999,"5min":1.996,"15min":5.6539999999999999},"last":1.04}]},{"osd":5,"last update":"Sun Jul 28 19:30:01 2024","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.96199999999999997,"15min":0.93100000000000005},"min":{"1min":0.75600000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":1.6040000000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.96999999999999997,"15min":0.95899999999999996},"min":{"1min":0.746,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.208,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.0149999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:29:53 2024","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.01,"15min":1.022},"min":{"1min":0.76000000000000001,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.1970000000000001,"5min":2.165,"15min":2.165},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.022,"5min":1.012,"15min":1.0069999999999999},"min":{"1min":0.77600000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.6619999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":1.1339999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:30:42 2024","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0389999999999999,"15min":1.077},"min":{"1min":0.66600000000000004,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2010000000000001,"5min":2.0880000000000001,"15min":2.0880000000000001},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.0409999999999999,"15min":1.071},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2090000000000001,"5min":1.704,"15min":1.704},"last":1.077}]}]},{"osd":2,"up_from":18,"seq":77309411447,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298080,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447072,"statfs":{"total":95995035648,"available":95689801728,"internally_reserved":0,"allocated":745472,"data_stored":324321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.61699999999999999,"15min":0.57899999999999996},"min":{"1min":0.39800000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":2.4569999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.79400000000000004,"15min":0.70599999999999996},"min":{"1min":0.47099999999999997,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":1.3049999999999999,"5min":1.456,"15min":1.456},"last":0.52900000000000003}]},{"osd":1,"last update":"Sun Jul 28 19:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.68500000000000005,"15min":0.61299999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.30599999999999999},"max":{"1min":1.1779999999999999,"5min":1.252,"15min":1.252},"last":0.505},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.745,"15min":0.67400000000000004},"min":{"1min":0.379,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":1.083,"5min":1.345,"15min":1.7350000000000001},"last":0.44800000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:30:27 2024","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.80500000000000005,"15min":0.76300000000000001},"min":{"1min":0.41699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.272,"5min":1.3759999999999999,"15min":1.9790000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.84099999999999997,"15min":0.80500000000000005},"min":{"1min":0.46700000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.224,"5min":1.248,"15min":1.605},"last":0.56699999999999995}]},{"osd":4,"last update":"Sun Jul 28 19:30:18 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.88500000000000001,"15min":0.85899999999999999},"min":{"1min":0.67100000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.98699999999999999,"15min":0.93300000000000005},"min":{"1min":0.67100000000000004,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.427,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.85399999999999998}]},{"osd":5,"last update":"Sun Jul 28 19:30:10 2024","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.94199999999999995,"15min":1.0209999999999999},"min":{"1min":0.73499999999999999,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.498,"5min":1.498,"15min":3.9969999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.95599999999999996,"15min":1.1020000000000001},"min":{"1min":0.68200000000000005,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.228,"5min":1.504,"15min":5.3680000000000003},"last":0.81000000000000005}]},{"osd":6,"last update":"Sun Jul 28 19:29:52 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1,"15min":1.0229999999999999},"min":{"1min":0.81399999999999995,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.3260000000000001,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.747},{"interface":"front","average":{"1min":0.997,"5min":1.048,"15min":1.048},"min":{"1min":0.70199999999999996,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.2649999999999999,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.879}]},{"osd":7,"last update":"Sun Jul 28 19:30:46 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.069,"15min":1.091},"min":{"1min":0.64500000000000002,"5min":0.621,"15min":0.621},"max":{"1min":1.4299999999999999,"5min":1.766,"15min":1.766},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0029999999999999,"15min":1.0289999999999999},"min":{"1min":0.60199999999999998,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.349,"5min":1.454,"15min":1.454},"last":0.77900000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574976,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":733184,"data_stored":323452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.65600000000000003,"15min":0.57599999999999996},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.84299999999999997,"5min":2.2530000000000001,"15min":2.2530000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.72599999999999998,"15min":0.63500000000000001},"min":{"1min":0.34000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.244,"5min":1.351,"15min":1.351},"last":0.68799999999999994}]},{"osd":2,"last update":"Sun Jul 28 19:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68500000000000005,"15min":0.61699999999999999},"min":{"1min":0.442,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":1.0089999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.77200000000000002,"15min":0.71299999999999997},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.0469999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.85099999999999998}]},{"osd":3,"last update":"Sun Jul 28 19:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.83299999999999996,"15min":0.79700000000000004},"min":{"1min":0.36899999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.254,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.129},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.753,"15min":0.751},"min":{"1min":0.29899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.069,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.89000000000000001}]},{"osd":4,"last update":"Sun Jul 28 19:30:12 2024","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.008,"15min":0.93500000000000005},"min":{"1min":0.73099999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.3069999999999999,"5min":1.554,"15min":1.554},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.93500000000000005,"15min":0.90600000000000003},"min":{"1min":0.74199999999999999,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3779999999999999,"5min":1.53,"15min":1.53},"last":0.98499999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:30:06 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94599999999999995,"15min":0.91400000000000003},"min":{"1min":0.68799999999999994,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.381,"5min":1.605,"15min":1.605},"last":1.2430000000000001},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.024,"15min":0.997},"min":{"1min":0.78900000000000003,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.355,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.0209999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:29:52 2024","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.087,"15min":1.089},"min":{"1min":0.73299999999999998,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.4139999999999999,"5min":1.74,"15min":1.74},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.054,"5min":1.0269999999999999,"15min":0.98699999999999999},"min":{"1min":0.67100000000000004,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.4199999999999999,"5min":1.778,"15min":1.778},"last":1.4019999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:30:50 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0229999999999999,"15min":1.0109999999999999},"min":{"1min":0.629,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.323,"5min":1.677,"15min":1.677},"last":1.323},{"interface":"front","average":{"1min":1.018,"5min":1.0329999999999999,"15min":1.0780000000000001},"min":{"1min":0.70799999999999996,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":1.2230000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738506,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298660,"kb_used_data":1308,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446492,"statfs":{"total":95995035648,"available":95689207808,"internally_reserved":0,"allocated":1339392,"data_stored":914677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Jul 28 19:30:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.63500000000000001,"15min":0.57099999999999995},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.014,"5min":1.0569999999999999,"15min":1.127},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.70699999999999996,"15min":0.63400000000000001},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.93700000000000006,"5min":1.347,"15min":1.347},"last":0.78300000000000003}]},{"osd":2,"last update":"Sun Jul 28 19:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.74099999999999999,"15min":0.68100000000000005},"min":{"1min":0.35899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.996,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.70199999999999996,"15min":0.67400000000000004},"min":{"1min":0.34399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.96099999999999997,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.73299999999999998}]},{"osd":3,"last update":"Sun Jul 28 19:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.91200000000000003,"5min":1.292,"15min":1.292},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.80500000000000005,"15min":0.81799999999999995},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.318},"max":{"1min":1.0669999999999999,"5min":1.177,"15min":1.3220000000000001},"last":0.86199999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:30:11 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.93300000000000005,"15min":0.85799999999999998},"min":{"1min":0.67600000000000005,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.2470000000000001,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.88},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.94999999999999996,"15min":0.90200000000000002},"min":{"1min":0.69599999999999995,"5min":0.437,"15min":0.437},"max":{"1min":1.1399999999999999,"5min":1.3,"15min":1.327},"last":1.0069999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.94099999999999995,"15min":0.93899999999999995},"min":{"1min":0.59299999999999997,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.2050000000000001,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.95099999999999996,"15min":0.92500000000000004},"min":{"1min":0.5,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.2310000000000001,"5min":1.385,"15min":1.385},"last":1.0580000000000001}]},{"osd":6,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.012,"15min":1.024},"min":{"1min":0.53800000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":1.1000000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0469999999999999,"15min":1.052},"min":{"1min":0.65100000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.5980000000000001,"5min":1.5980000000000001,"15min":1.5980000000000001},"last":1.03}]},{"osd":7,"last update":"Sun Jul 28 19:30:49 2024","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.0369999999999999,"15min":1.0229999999999999},"min":{"1min":0.60699999999999998,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":6.2489999999999997,"5min":6.2489999999999997,"15min":6.2489999999999997},"last":1.1439999999999999},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.004,"15min":1.0349999999999999},"min":{"1min":0.55800000000000005,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.3520000000000001,"5min":1.573,"15min":1.573},"last":1.2050000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-28T19:30:55.504 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph pg dump --format=json 2024-07-28T19:30:56.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:56 smithi100 bash[21828]: audit 2024-07-28T19:30:54.804638+0000 mgr.y (mgr.14391) 171 : audit [DBG] from='client.14739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:30:56.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:56 smithi100 bash[21828]: cluster 2024-07-28T19:30:54.906786+0000 mgr.y (mgr.14391) 172 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:56.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:56 smithi100 bash[31715]: audit 2024-07-28T19:30:54.804638+0000 mgr.y (mgr.14391) 171 : audit [DBG] from='client.14739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:30:56.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:56 smithi100 bash[31715]: cluster 2024-07-28T19:30:54.906786+0000 mgr.y (mgr.14391) 172 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:56.553 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:30:56 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:30:56.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:56 smithi138 bash[23952]: audit 2024-07-28T19:30:54.804638+0000 mgr.y (mgr.14391) 171 : audit [DBG] from='client.14739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:30:56.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:56 smithi138 bash[23952]: cluster 2024-07-28T19:30:54.906786+0000 mgr.y (mgr.14391) 172 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:58.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:30:58 smithi100 bash[21828]: cluster 2024-07-28T19:30:56.907757+0000 mgr.y (mgr.14391) 173 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:58.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:30:58 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:30:58] "GET /metrics HTTP/1.1" 200 239409 "" "Prometheus/2.43.0" 2024-07-28T19:30:58.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:30:58 smithi100 bash[31715]: cluster 2024-07-28T19:30:56.907757+0000 mgr.y (mgr.14391) 173 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:30:58.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:30:58 smithi138 bash[23952]: cluster 2024-07-28T19:30:56.907757+0000 mgr.y (mgr.14391) 173 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:00.280 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:31:00.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:00 smithi100 bash[21828]: cluster 2024-07-28T19:30:58.908424+0000 mgr.y (mgr.14391) 174 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:00.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:00 smithi100 bash[31715]: cluster 2024-07-28T19:30:58.908424+0000 mgr.y (mgr.14391) 174 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:00.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:00 smithi138 bash[23952]: cluster 2024-07-28T19:30:58.908424+0000 mgr.y (mgr.14391) 174 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:01.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:01 smithi100 bash[21828]: audit 2024-07-28T19:31:00.910158+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:01.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:01 smithi100 bash[31715]: audit 2024-07-28T19:31:00.910158+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:01.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:01 smithi138 bash[23952]: audit 2024-07-28T19:31:00.910158+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:02.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:02 smithi100 bash[21828]: cluster 2024-07-28T19:31:00.908969+0000 mgr.y (mgr.14391) 175 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:02.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:02 smithi100 bash[31715]: cluster 2024-07-28T19:31:00.908969+0000 mgr.y (mgr.14391) 175 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:02.553 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:02 smithi138 bash[23952]: cluster 2024-07-28T19:31:00.908969+0000 mgr.y (mgr.14391) 175 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:03.532 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:31:03.534 INFO:teuthology.orchestra.run.smithi100.stderr:dumped all 2024-07-28T19:31:04.172 INFO:teuthology.orchestra.run.smithi100.stdout:{"pg_ready":true,"pg_map":{"version":151,"stamp":"2024-07-28T19:31:02.909256+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":259,"num_read_kb":355,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386420,"kb_used_data":7576,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574796,"statfs":{"total":767960285184,"available":765516591104,"internally_reserved":0,"allocated":7757824,"data_stored":4366413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003708"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-07-28T19:29:32.082110+0000","last_change":"2024-07-28T19:29:32.082110+0000","last_active":"2024-07-28T19:29:32.082110+0000","last_peered":"2024-07-28T19:29:32.082110+0000","last_clean":"2024-07-28T19:29:32.082110+0000","last_became_active":"2024-07-28T19:29:32.081261+0000","last_became_peered":"2024-07-28T19:29:32.081261+0000","last_unstale":"2024-07-28T19:29:32.082110+0000","last_undegraded":"2024-07-28T19:29:32.082110+0000","last_fullsized":"2024-07-28T19:29:32.082110+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:05:16.608598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.946643+0000","last_change":"2024-07-28T19:29:15.300221+0000","last_active":"2024-07-28T19:29:30.946643+0000","last_peered":"2024-07-28T19:29:30.946643+0000","last_clean":"2024-07-28T19:29:30.946643+0000","last_became_active":"2024-07-28T19:29:15.298656+0000","last_became_peered":"2024-07-28T19:29:15.298656+0000","last_unstale":"2024-07-28T19:29:30.946643+0000","last_undegraded":"2024-07-28T19:29:30.946643+0000","last_fullsized":"2024-07-28T19:29:30.946643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:00:31.428501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.950975+0000","last_change":"2024-07-28T19:29:16.447233+0000","last_active":"2024-07-28T19:29:30.950975+0000","last_peered":"2024-07-28T19:29:30.950975+0000","last_clean":"2024-07-28T19:29:30.950975+0000","last_became_active":"2024-07-28T19:29:16.446518+0000","last_became_peered":"2024-07-28T19:29:16.446518+0000","last_unstale":"2024-07-28T19:29:30.950975+0000","last_undegraded":"2024-07-28T19:29:30.950975+0000","last_fullsized":"2024-07-28T19:29:30.950975+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:34:35.744242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.948944+0000","last_change":"2024-07-28T19:29:15.283186+0000","last_active":"2024-07-28T19:29:30.948944+0000","last_peered":"2024-07-28T19:29:30.948944+0000","last_clean":"2024-07-28T19:29:30.948944+0000","last_became_active":"2024-07-28T19:29:15.282685+0000","last_became_peered":"2024-07-28T19:29:15.282685+0000","last_unstale":"2024-07-28T19:29:30.948944+0000","last_undegraded":"2024-07-28T19:29:30.948944+0000","last_fullsized":"2024-07-28T19:29:30.948944+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:06:18.152760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.948973+0000","last_change":"2024-07-28T19:29:16.450398+0000","last_active":"2024-07-28T19:29:30.948973+0000","last_peered":"2024-07-28T19:29:30.948973+0000","last_clean":"2024-07-28T19:29:30.948973+0000","last_became_active":"2024-07-28T19:29:16.449594+0000","last_became_peered":"2024-07-28T19:29:16.449594+0000","last_unstale":"2024-07-28T19:29:30.948973+0000","last_undegraded":"2024-07-28T19:29:30.948973+0000","last_fullsized":"2024-07-28T19:29:30.948973+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:28:35.449786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.946639+0000","last_change":"2024-07-28T19:29:15.299547+0000","last_active":"2024-07-28T19:29:30.946639+0000","last_peered":"2024-07-28T19:29:30.946639+0000","last_clean":"2024-07-28T19:29:30.946639+0000","last_became_active":"2024-07-28T19:29:15.296937+0000","last_became_peered":"2024-07-28T19:29:15.296937+0000","last_unstale":"2024-07-28T19:29:30.946639+0000","last_undegraded":"2024-07-28T19:29:30.946639+0000","last_fullsized":"2024-07-28T19:29:30.946639+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:03:30.991662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958179+0000","last_change":"2024-07-28T19:29:16.448600+0000","last_active":"2024-07-28T19:29:30.958179+0000","last_peered":"2024-07-28T19:29:30.958179+0000","last_clean":"2024-07-28T19:29:30.958179+0000","last_became_active":"2024-07-28T19:29:16.447856+0000","last_became_peered":"2024-07-28T19:29:16.447856+0000","last_unstale":"2024-07-28T19:29:30.958179+0000","last_undegraded":"2024-07-28T19:29:30.958179+0000","last_fullsized":"2024-07-28T19:29:30.958179+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:18:48.990747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":27,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.600532+0000","last_change":"2024-07-28T19:29:15.264709+0000","last_active":"2024-07-28T19:29:31.600532+0000","last_peered":"2024-07-28T19:29:31.600532+0000","last_clean":"2024-07-28T19:29:31.600532+0000","last_became_active":"2024-07-28T19:29:15.261437+0000","last_became_peered":"2024-07-28T19:29:15.261437+0000","last_unstale":"2024-07-28T19:29:31.600532+0000","last_undegraded":"2024-07-28T19:29:31.600532+0000","last_fullsized":"2024-07-28T19:29:31.600532+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:22:34.843162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947782+0000","last_change":"2024-07-28T19:29:15.299396+0000","last_active":"2024-07-28T19:29:30.947782+0000","last_peered":"2024-07-28T19:29:30.947782+0000","last_clean":"2024-07-28T19:29:30.947782+0000","last_became_active":"2024-07-28T19:29:15.297605+0000","last_became_peered":"2024-07-28T19:29:15.297605+0000","last_unstale":"2024-07-28T19:29:30.947782+0000","last_undegraded":"2024-07-28T19:29:30.947782+0000","last_fullsized":"2024-07-28T19:29:30.947782+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:32:23.182916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.950776+0000","last_change":"2024-07-28T19:29:16.445468+0000","last_active":"2024-07-28T19:29:30.950776+0000","last_peered":"2024-07-28T19:29:30.950776+0000","last_clean":"2024-07-28T19:29:30.950776+0000","last_became_active":"2024-07-28T19:29:16.444693+0000","last_became_peered":"2024-07-28T19:29:16.444693+0000","last_unstale":"2024-07-28T19:29:30.950776+0000","last_undegraded":"2024-07-28T19:29:30.950776+0000","last_fullsized":"2024-07-28T19:29:30.950776+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:13:39.927285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947561+0000","last_change":"2024-07-28T19:29:15.300758+0000","last_active":"2024-07-28T19:29:30.947561+0000","last_peered":"2024-07-28T19:29:30.947561+0000","last_clean":"2024-07-28T19:29:30.947561+0000","last_became_active":"2024-07-28T19:29:15.299775+0000","last_became_peered":"2024-07-28T19:29:15.299775+0000","last_unstale":"2024-07-28T19:29:30.947561+0000","last_undegraded":"2024-07-28T19:29:30.947561+0000","last_fullsized":"2024-07-28T19:29:30.947561+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:50:19.096556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957388+0000","last_change":"2024-07-28T19:29:15.299875+0000","last_active":"2024-07-28T19:29:30.957388+0000","last_peered":"2024-07-28T19:29:30.957388+0000","last_clean":"2024-07-28T19:29:30.957388+0000","last_became_active":"2024-07-28T19:29:15.299085+0000","last_became_peered":"2024-07-28T19:29:15.299085+0000","last_unstale":"2024-07-28T19:29:30.957388+0000","last_undegraded":"2024-07-28T19:29:30.957388+0000","last_fullsized":"2024-07-28T19:29:30.957388+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:05:00.231080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947481+0000","last_change":"2024-07-28T19:29:15.279653+0000","last_active":"2024-07-28T19:29:30.947481+0000","last_peered":"2024-07-28T19:29:30.947481+0000","last_clean":"2024-07-28T19:29:30.947481+0000","last_became_active":"2024-07-28T19:29:15.278829+0000","last_became_peered":"2024-07-28T19:29:15.278829+0000","last_unstale":"2024-07-28T19:29:30.947481+0000","last_undegraded":"2024-07-28T19:29:30.947481+0000","last_fullsized":"2024-07-28T19:29:30.947481+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:46:07.127995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.602044+0000","last_change":"2024-07-28T19:29:16.445265+0000","last_active":"2024-07-28T19:29:31.602044+0000","last_peered":"2024-07-28T19:29:31.602044+0000","last_clean":"2024-07-28T19:29:31.602044+0000","last_became_active":"2024-07-28T19:29:16.444411+0000","last_became_peered":"2024-07-28T19:29:16.444411+0000","last_unstale":"2024-07-28T19:29:31.602044+0000","last_undegraded":"2024-07-28T19:29:31.602044+0000","last_fullsized":"2024-07-28T19:29:31.602044+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:32:45.312229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.951048+0000","last_change":"2024-07-28T19:29:15.267503+0000","last_active":"2024-07-28T19:29:30.951048+0000","last_peered":"2024-07-28T19:29:30.951048+0000","last_clean":"2024-07-28T19:29:30.951048+0000","last_became_active":"2024-07-28T19:29:15.266632+0000","last_became_peered":"2024-07-28T19:29:15.266632+0000","last_unstale":"2024-07-28T19:29:30.951048+0000","last_undegraded":"2024-07-28T19:29:30.951048+0000","last_fullsized":"2024-07-28T19:29:30.951048+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:41:34.344331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949118+0000","last_change":"2024-07-28T19:29:16.325716+0000","last_active":"2024-07-28T19:29:30.949118+0000","last_peered":"2024-07-28T19:29:30.949118+0000","last_clean":"2024-07-28T19:29:30.949118+0000","last_became_active":"2024-07-28T19:29:16.325194+0000","last_became_peered":"2024-07-28T19:29:16.325194+0000","last_unstale":"2024-07-28T19:29:30.949118+0000","last_undegraded":"2024-07-28T19:29:30.949118+0000","last_fullsized":"2024-07-28T19:29:30.949118+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:27:16.454707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.956318+0000","last_change":"2024-07-28T19:29:15.304133+0000","last_active":"2024-07-28T19:29:30.956318+0000","last_peered":"2024-07-28T19:29:30.956318+0000","last_clean":"2024-07-28T19:29:30.956318+0000","last_became_active":"2024-07-28T19:29:15.303800+0000","last_became_peered":"2024-07-28T19:29:15.303800+0000","last_unstale":"2024-07-28T19:29:30.956318+0000","last_undegraded":"2024-07-28T19:29:30.956318+0000","last_fullsized":"2024-07-28T19:29:30.956318+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:44:18.395540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957700+0000","last_change":"2024-07-28T19:29:15.307023+0000","last_active":"2024-07-28T19:29:30.957700+0000","last_peered":"2024-07-28T19:29:30.957700+0000","last_clean":"2024-07-28T19:29:30.957700+0000","last_became_active":"2024-07-28T19:29:15.306567+0000","last_became_peered":"2024-07-28T19:29:15.306567+0000","last_unstale":"2024-07-28T19:29:30.957700+0000","last_undegraded":"2024-07-28T19:29:30.957700+0000","last_fullsized":"2024-07-28T19:29:30.957700+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:06:31.551224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.600619+0000","last_change":"2024-07-28T19:29:15.280359+0000","last_active":"2024-07-28T19:29:31.600619+0000","last_peered":"2024-07-28T19:29:31.600619+0000","last_clean":"2024-07-28T19:29:31.600619+0000","last_became_active":"2024-07-28T19:29:15.279580+0000","last_became_peered":"2024-07-28T19:29:15.279580+0000","last_unstale":"2024-07-28T19:29:31.600619+0000","last_undegraded":"2024-07-28T19:29:31.600619+0000","last_fullsized":"2024-07-28T19:29:31.600619+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:07:48.900003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957978+0000","last_change":"2024-07-28T19:29:15.291033+0000","last_active":"2024-07-28T19:29:30.957978+0000","last_peered":"2024-07-28T19:29:30.957978+0000","last_clean":"2024-07-28T19:29:30.957978+0000","last_became_active":"2024-07-28T19:29:15.290414+0000","last_became_peered":"2024-07-28T19:29:15.290414+0000","last_unstale":"2024-07-28T19:29:30.957978+0000","last_undegraded":"2024-07-28T19:29:30.957978+0000","last_fullsized":"2024-07-28T19:29:30.957978+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:34:33.307390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957455+0000","last_change":"2024-07-28T19:29:15.259681+0000","last_active":"2024-07-28T19:29:30.957455+0000","last_peered":"2024-07-28T19:29:30.957455+0000","last_clean":"2024-07-28T19:29:30.957455+0000","last_became_active":"2024-07-28T19:29:15.258936+0000","last_became_peered":"2024-07-28T19:29:15.258936+0000","last_unstale":"2024-07-28T19:29:30.957455+0000","last_undegraded":"2024-07-28T19:29:30.957455+0000","last_fullsized":"2024-07-28T19:29:30.957455+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:47:44.039720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958162+0000","last_change":"2024-07-28T19:29:15.304430+0000","last_active":"2024-07-28T19:29:30.958162+0000","last_peered":"2024-07-28T19:29:30.958162+0000","last_clean":"2024-07-28T19:29:30.958162+0000","last_became_active":"2024-07-28T19:29:15.304053+0000","last_became_peered":"2024-07-28T19:29:15.304053+0000","last_unstale":"2024-07-28T19:29:30.958162+0000","last_undegraded":"2024-07-28T19:29:30.958162+0000","last_fullsized":"2024-07-28T19:29:30.958162+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:55:25.885784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955802+0000","last_change":"2024-07-28T19:29:15.280675+0000","last_active":"2024-07-28T19:29:30.955802+0000","last_peered":"2024-07-28T19:29:30.955802+0000","last_clean":"2024-07-28T19:29:30.955802+0000","last_became_active":"2024-07-28T19:29:15.280312+0000","last_became_peered":"2024-07-28T19:29:15.280312+0000","last_unstale":"2024-07-28T19:29:30.955802+0000","last_undegraded":"2024-07-28T19:29:30.955802+0000","last_fullsized":"2024-07-28T19:29:30.955802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:22:09.192184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.959576+0000","last_change":"2024-07-28T19:29:15.298989+0000","last_active":"2024-07-28T19:29:30.959576+0000","last_peered":"2024-07-28T19:29:30.959576+0000","last_clean":"2024-07-28T19:29:30.959576+0000","last_became_active":"2024-07-28T19:29:15.298703+0000","last_became_peered":"2024-07-28T19:29:15.298703+0000","last_unstale":"2024-07-28T19:29:30.959576+0000","last_undegraded":"2024-07-28T19:29:30.959576+0000","last_fullsized":"2024-07-28T19:29:30.959576+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:20:28.312665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":278,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.946869+0000","last_change":"2024-07-28T19:25:38.028024+0000","last_active":"2024-07-28T19:29:30.946869+0000","last_peered":"2024-07-28T19:29:30.946869+0000","last_clean":"2024-07-28T19:29:30.946869+0000","last_became_active":"2024-07-28T19:25:38.027110+0000","last_became_peered":"2024-07-28T19:25:38.027110+0000","last_unstale":"2024-07-28T19:29:30.946869+0000","last_undegraded":"2024-07-28T19:29:30.946869+0000","last_fullsized":"2024-07-28T19:29:30.946869+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_clean_scrub_stamp":"2024-07-28T19:21:20.215133+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:20:35.538874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.951021+0000","last_change":"2024-07-28T19:27:15.550979+0000","last_active":"2024-07-28T19:29:30.951021+0000","last_peered":"2024-07-28T19:29:30.951021+0000","last_clean":"2024-07-28T19:29:30.951021+0000","last_became_active":"2024-07-28T19:27:15.550214+0000","last_became_peered":"2024-07-28T19:27:15.550214+0000","last_unstale":"2024-07-28T19:29:30.951021+0000","last_undegraded":"2024-07-28T19:29:30.951021+0000","last_fullsized":"2024-07-28T19:29:30.951021+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:17:06.298582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955877+0000","last_change":"2024-07-28T19:29:15.262671+0000","last_active":"2024-07-28T19:29:30.955877+0000","last_peered":"2024-07-28T19:29:30.955877+0000","last_clean":"2024-07-28T19:29:30.955877+0000","last_became_active":"2024-07-28T19:29:15.262218+0000","last_became_peered":"2024-07-28T19:29:15.262218+0000","last_unstale":"2024-07-28T19:29:30.955877+0000","last_undegraded":"2024-07-28T19:29:30.955877+0000","last_fullsized":"2024-07-28T19:29:30.955877+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:41:19.915904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.601607+0000","last_change":"2024-07-28T19:29:16.446131+0000","last_active":"2024-07-28T19:29:31.601607+0000","last_peered":"2024-07-28T19:29:31.601607+0000","last_clean":"2024-07-28T19:29:31.601607+0000","last_became_active":"2024-07-28T19:29:16.445372+0000","last_became_peered":"2024-07-28T19:29:16.445372+0000","last_unstale":"2024-07-28T19:29:31.601607+0000","last_undegraded":"2024-07-28T19:29:31.601607+0000","last_fullsized":"2024-07-28T19:29:31.601607+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:38:05.866961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.946641+0000","last_change":"2024-07-28T19:27:15.702756+0000","last_active":"2024-07-28T19:29:30.946641+0000","last_peered":"2024-07-28T19:29:30.946641+0000","last_clean":"2024-07-28T19:29:30.946641+0000","last_became_active":"2024-07-28T19:27:15.701931+0000","last_became_peered":"2024-07-28T19:27:15.701931+0000","last_unstale":"2024-07-28T19:29:30.946641+0000","last_undegraded":"2024-07-28T19:29:30.946641+0000","last_fullsized":"2024-07-28T19:29:30.946641+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:20:20.149727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.948369+0000","last_change":"2024-07-28T19:29:15.281513+0000","last_active":"2024-07-28T19:29:30.948369+0000","last_peered":"2024-07-28T19:29:30.948369+0000","last_clean":"2024-07-28T19:29:30.948369+0000","last_became_active":"2024-07-28T19:29:15.280706+0000","last_became_peered":"2024-07-28T19:29:15.280706+0000","last_unstale":"2024-07-28T19:29:30.948369+0000","last_undegraded":"2024-07-28T19:29:30.948369+0000","last_fullsized":"2024-07-28T19:29:30.948369+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:07:49.929016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-07-28T19:29:32.517058+0000","last_change":"2024-07-28T19:29:32.517058+0000","last_active":"2024-07-28T19:29:32.517058+0000","last_peered":"2024-07-28T19:29:32.517058+0000","last_clean":"2024-07-28T19:29:32.517058+0000","last_became_active":"2024-07-28T19:29:32.516140+0000","last_became_peered":"2024-07-28T19:29:32.516140+0000","last_unstale":"2024-07-28T19:29:32.517058+0000","last_undegraded":"2024-07-28T19:29:32.517058+0000","last_fullsized":"2024-07-28T19:29:32.517058+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:54:48.068694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955310+0000","last_change":"2024-07-28T19:27:15.706055+0000","last_active":"2024-07-28T19:29:30.955310+0000","last_peered":"2024-07-28T19:29:30.955310+0000","last_clean":"2024-07-28T19:29:30.955310+0000","last_became_active":"2024-07-28T19:27:15.705102+0000","last_became_peered":"2024-07-28T19:27:15.705102+0000","last_unstale":"2024-07-28T19:29:30.955310+0000","last_undegraded":"2024-07-28T19:29:30.955310+0000","last_fullsized":"2024-07-28T19:29:30.955310+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:00:20.179809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.953902+0000","last_change":"2024-07-28T19:29:15.259870+0000","last_active":"2024-07-28T19:29:30.953902+0000","last_peered":"2024-07-28T19:29:30.953902+0000","last_clean":"2024-07-28T19:29:30.953902+0000","last_became_active":"2024-07-28T19:29:15.259169+0000","last_became_peered":"2024-07-28T19:29:15.259169+0000","last_unstale":"2024-07-28T19:29:30.953902+0000","last_undegraded":"2024-07-28T19:29:30.953902+0000","last_fullsized":"2024-07-28T19:29:30.953902+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:53:53.614089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.964769+0000","last_change":"2024-07-28T19:29:15.603688+0000","last_active":"2024-07-28T19:29:30.964769+0000","last_peered":"2024-07-28T19:29:30.964769+0000","last_clean":"2024-07-28T19:29:30.964769+0000","last_became_active":"2024-07-28T19:29:15.603137+0000","last_became_peered":"2024-07-28T19:29:15.603137+0000","last_unstale":"2024-07-28T19:29:30.964769+0000","last_undegraded":"2024-07-28T19:29:30.964769+0000","last_fullsized":"2024-07-28T19:29:30.964769+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:40:41.988368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.951086+0000","last_change":"2024-07-28T19:27:15.552985+0000","last_active":"2024-07-28T19:29:30.951086+0000","last_peered":"2024-07-28T19:29:30.951086+0000","last_clean":"2024-07-28T19:29:30.951086+0000","last_became_active":"2024-07-28T19:27:15.552104+0000","last_became_peered":"2024-07-28T19:27:15.552104+0000","last_unstale":"2024-07-28T19:29:30.951086+0000","last_undegraded":"2024-07-28T19:29:30.951086+0000","last_fullsized":"2024-07-28T19:29:30.951086+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:46:22.374425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957804+0000","last_change":"2024-07-28T19:29:15.278523+0000","last_active":"2024-07-28T19:29:30.957804+0000","last_peered":"2024-07-28T19:29:30.957804+0000","last_clean":"2024-07-28T19:29:30.957804+0000","last_became_active":"2024-07-28T19:29:15.277560+0000","last_became_peered":"2024-07-28T19:29:15.277560+0000","last_unstale":"2024-07-28T19:29:30.957804+0000","last_undegraded":"2024-07-28T19:29:30.957804+0000","last_fullsized":"2024-07-28T19:29:30.957804+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T06:26:39.820338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.960654+0000","last_change":"2024-07-28T19:29:15.297935+0000","last_active":"2024-07-28T19:29:30.960654+0000","last_peered":"2024-07-28T19:29:30.960654+0000","last_clean":"2024-07-28T19:29:30.960654+0000","last_became_active":"2024-07-28T19:29:15.297583+0000","last_became_peered":"2024-07-28T19:29:15.297583+0000","last_unstale":"2024-07-28T19:29:30.960654+0000","last_undegraded":"2024-07-28T19:29:30.960654+0000","last_fullsized":"2024-07-28T19:29:30.960654+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:47:11.793168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.952020+0000","last_change":"2024-07-28T19:29:15.266697+0000","last_active":"2024-07-28T19:29:30.952020+0000","last_peered":"2024-07-28T19:29:30.952020+0000","last_clean":"2024-07-28T19:29:30.952020+0000","last_became_active":"2024-07-28T19:29:15.265912+0000","last_became_peered":"2024-07-28T19:29:15.265912+0000","last_unstale":"2024-07-28T19:29:30.952020+0000","last_undegraded":"2024-07-28T19:29:30.952020+0000","last_fullsized":"2024-07-28T19:29:30.952020+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:03:55.516994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.960929+0000","last_change":"2024-07-28T19:27:15.706361+0000","last_active":"2024-07-28T19:29:30.960929+0000","last_peered":"2024-07-28T19:29:30.960929+0000","last_clean":"2024-07-28T19:29:30.960929+0000","last_became_active":"2024-07-28T19:27:15.705499+0000","last_became_peered":"2024-07-28T19:27:15.705499+0000","last_unstale":"2024-07-28T19:29:30.960929+0000","last_undegraded":"2024-07-28T19:29:30.960929+0000","last_fullsized":"2024-07-28T19:29:30.960929+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:11:20.803138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-07-28T19:29:32.516389+0000","last_change":"2024-07-28T19:29:32.516389+0000","last_active":"2024-07-28T19:29:32.516389+0000","last_peered":"2024-07-28T19:29:32.516389+0000","last_clean":"2024-07-28T19:29:32.516389+0000","last_became_active":"2024-07-28T19:29:32.515454+0000","last_became_peered":"2024-07-28T19:29:32.515454+0000","last_unstale":"2024-07-28T19:29:32.516389+0000","last_undegraded":"2024-07-28T19:29:32.516389+0000","last_fullsized":"2024-07-28T19:29:32.516389+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:44:35.842708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.960797+0000","last_change":"2024-07-28T19:29:15.267962+0000","last_active":"2024-07-28T19:29:30.960797+0000","last_peered":"2024-07-28T19:29:30.960797+0000","last_clean":"2024-07-28T19:29:30.960797+0000","last_became_active":"2024-07-28T19:29:15.267350+0000","last_became_peered":"2024-07-28T19:29:15.267350+0000","last_unstale":"2024-07-28T19:29:30.960797+0000","last_undegraded":"2024-07-28T19:29:30.960797+0000","last_fullsized":"2024-07-28T19:29:30.960797+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:14:25.642811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947464+0000","last_change":"2024-07-28T19:27:15.703022+0000","last_active":"2024-07-28T19:29:30.947464+0000","last_peered":"2024-07-28T19:29:30.947464+0000","last_clean":"2024-07-28T19:29:30.947464+0000","last_became_active":"2024-07-28T19:27:15.701965+0000","last_became_peered":"2024-07-28T19:27:15.701965+0000","last_unstale":"2024-07-28T19:29:30.947464+0000","last_undegraded":"2024-07-28T19:29:30.947464+0000","last_fullsized":"2024-07-28T19:29:30.947464+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:18:56.026871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949263+0000","last_change":"2024-07-28T19:29:16.448067+0000","last_active":"2024-07-28T19:29:30.949263+0000","last_peered":"2024-07-28T19:29:30.949263+0000","last_clean":"2024-07-28T19:29:30.949263+0000","last_became_active":"2024-07-28T19:29:16.447172+0000","last_became_peered":"2024-07-28T19:29:16.447172+0000","last_unstale":"2024-07-28T19:29:30.949263+0000","last_undegraded":"2024-07-28T19:29:30.949263+0000","last_fullsized":"2024-07-28T19:29:30.949263+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:53:50.969459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958440+0000","last_change":"2024-07-28T19:29:15.281381+0000","last_active":"2024-07-28T19:29:30.958440+0000","last_peered":"2024-07-28T19:29:30.958440+0000","last_clean":"2024-07-28T19:29:30.958440+0000","last_became_active":"2024-07-28T19:29:15.280586+0000","last_became_peered":"2024-07-28T19:29:15.280586+0000","last_unstale":"2024-07-28T19:29:30.958440+0000","last_undegraded":"2024-07-28T19:29:30.958440+0000","last_fullsized":"2024-07-28T19:29:30.958440+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:17:11.213449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.961792+0000","last_change":"2024-07-28T19:27:15.704938+0000","last_active":"2024-07-28T19:29:30.961792+0000","last_peered":"2024-07-28T19:29:30.961792+0000","last_clean":"2024-07-28T19:29:30.961792+0000","last_became_active":"2024-07-28T19:27:15.703726+0000","last_became_peered":"2024-07-28T19:27:15.703726+0000","last_unstale":"2024-07-28T19:29:30.961792+0000","last_undegraded":"2024-07-28T19:29:30.961792+0000","last_fullsized":"2024-07-28T19:29:30.961792+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:39:26.136768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955363+0000","last_change":"2024-07-28T19:29:16.454445+0000","last_active":"2024-07-28T19:29:30.955363+0000","last_peered":"2024-07-28T19:29:30.955363+0000","last_clean":"2024-07-28T19:29:30.955363+0000","last_became_active":"2024-07-28T19:29:16.453274+0000","last_became_peered":"2024-07-28T19:29:16.453274+0000","last_unstale":"2024-07-28T19:29:30.955363+0000","last_undegraded":"2024-07-28T19:29:30.955363+0000","last_fullsized":"2024-07-28T19:29:30.955363+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:43:40.314510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949287+0000","last_change":"2024-07-28T19:29:15.278341+0000","last_active":"2024-07-28T19:29:30.949287+0000","last_peered":"2024-07-28T19:29:30.949287+0000","last_clean":"2024-07-28T19:29:30.949287+0000","last_became_active":"2024-07-28T19:29:15.277543+0000","last_became_peered":"2024-07-28T19:29:15.277543+0000","last_unstale":"2024-07-28T19:29:30.949287+0000","last_undegraded":"2024-07-28T19:29:30.949287+0000","last_fullsized":"2024-07-28T19:29:30.949287+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:31:05.176791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.600441+0000","last_change":"2024-07-28T19:27:15.560013+0000","last_active":"2024-07-28T19:29:31.600441+0000","last_peered":"2024-07-28T19:29:31.600441+0000","last_clean":"2024-07-28T19:29:31.600441+0000","last_became_active":"2024-07-28T19:27:15.559241+0000","last_became_peered":"2024-07-28T19:27:15.559241+0000","last_unstale":"2024-07-28T19:29:31.600441+0000","last_undegraded":"2024-07-28T19:29:31.600441+0000","last_fullsized":"2024-07-28T19:29:31.600441+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:41:17.422708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957169+0000","last_change":"2024-07-28T19:29:15.303436+0000","last_active":"2024-07-28T19:29:30.957169+0000","last_peered":"2024-07-28T19:29:30.957169+0000","last_clean":"2024-07-28T19:29:30.957169+0000","last_became_active":"2024-07-28T19:29:15.302771+0000","last_became_peered":"2024-07-28T19:29:15.302771+0000","last_unstale":"2024-07-28T19:29:30.957169+0000","last_undegraded":"2024-07-28T19:29:30.957169+0000","last_fullsized":"2024-07-28T19:29:30.957169+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:27:55.834358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957585+0000","last_change":"2024-07-28T19:29:15.303958+0000","last_active":"2024-07-28T19:29:30.957585+0000","last_peered":"2024-07-28T19:29:30.957585+0000","last_clean":"2024-07-28T19:29:30.957585+0000","last_became_active":"2024-07-28T19:29:15.303509+0000","last_became_peered":"2024-07-28T19:29:15.303509+0000","last_unstale":"2024-07-28T19:29:30.957585+0000","last_undegraded":"2024-07-28T19:29:30.957585+0000","last_fullsized":"2024-07-28T19:29:30.957585+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:18:55.497305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"70'7","reported_seq":27,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957823+0000","last_change":"2024-07-28T19:29:15.281929+0000","last_active":"2024-07-28T19:29:30.957823+0000","last_peered":"2024-07-28T19:29:30.957823+0000","last_clean":"2024-07-28T19:29:30.957823+0000","last_became_active":"2024-07-28T19:29:15.277637+0000","last_became_peered":"2024-07-28T19:29:15.277637+0000","last_unstale":"2024-07-28T19:29:30.957823+0000","last_undegraded":"2024-07-28T19:29:30.957823+0000","last_fullsized":"2024-07-28T19:29:30.957823+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:04:03.599599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.959794+0000","last_change":"2024-07-28T19:29:15.296240+0000","last_active":"2024-07-28T19:29:30.959794+0000","last_peered":"2024-07-28T19:29:30.959794+0000","last_clean":"2024-07-28T19:29:30.959794+0000","last_became_active":"2024-07-28T19:29:15.295869+0000","last_became_peered":"2024-07-28T19:29:15.295869+0000","last_unstale":"2024-07-28T19:29:30.959794+0000","last_undegraded":"2024-07-28T19:29:30.959794+0000","last_fullsized":"2024-07-28T19:29:30.959794+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:12:38.976248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.947195+0000","last_change":"2024-07-28T19:29:15.299299+0000","last_active":"2024-07-28T19:29:30.947195+0000","last_peered":"2024-07-28T19:29:30.947195+0000","last_clean":"2024-07-28T19:29:30.947195+0000","last_became_active":"2024-07-28T19:29:15.296865+0000","last_became_peered":"2024-07-28T19:29:15.296865+0000","last_unstale":"2024-07-28T19:29:30.947195+0000","last_undegraded":"2024-07-28T19:29:30.947195+0000","last_fullsized":"2024-07-28T19:29:30.947195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:54:46.661711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.952007+0000","last_change":"2024-07-28T19:29:15.277680+0000","last_active":"2024-07-28T19:29:30.952007+0000","last_peered":"2024-07-28T19:29:30.952007+0000","last_clean":"2024-07-28T19:29:30.952007+0000","last_became_active":"2024-07-28T19:29:15.276939+0000","last_became_peered":"2024-07-28T19:29:15.276939+0000","last_unstale":"2024-07-28T19:29:30.952007+0000","last_undegraded":"2024-07-28T19:29:30.952007+0000","last_fullsized":"2024-07-28T19:29:30.952007+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:39:43.008228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955925+0000","last_change":"2024-07-28T19:29:16.447973+0000","last_active":"2024-07-28T19:29:30.955925+0000","last_peered":"2024-07-28T19:29:30.955925+0000","last_clean":"2024-07-28T19:29:30.955925+0000","last_became_active":"2024-07-28T19:29:16.447208+0000","last_became_peered":"2024-07-28T19:29:16.447208+0000","last_unstale":"2024-07-28T19:29:30.955925+0000","last_undegraded":"2024-07-28T19:29:30.955925+0000","last_fullsized":"2024-07-28T19:29:30.955925+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:16:12.017505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949716+0000","last_change":"2024-07-28T19:29:15.284313+0000","last_active":"2024-07-28T19:29:30.949716+0000","last_peered":"2024-07-28T19:29:30.949716+0000","last_clean":"2024-07-28T19:29:30.949716+0000","last_became_active":"2024-07-28T19:29:15.283919+0000","last_became_peered":"2024-07-28T19:29:15.283919+0000","last_unstale":"2024-07-28T19:29:30.949716+0000","last_undegraded":"2024-07-28T19:29:30.949716+0000","last_fullsized":"2024-07-28T19:29:30.949716+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:18:48.974717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955152+0000","last_change":"2024-07-28T19:29:16.293216+0000","last_active":"2024-07-28T19:29:30.955152+0000","last_peered":"2024-07-28T19:29:30.955152+0000","last_clean":"2024-07-28T19:29:30.955152+0000","last_became_active":"2024-07-28T19:29:16.292635+0000","last_became_peered":"2024-07-28T19:29:16.292635+0000","last_unstale":"2024-07-28T19:29:30.955152+0000","last_undegraded":"2024-07-28T19:29:30.955152+0000","last_fullsized":"2024-07-28T19:29:30.955152+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:09:58.615631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:31.601878+0000","last_change":"2024-07-28T19:29:16.276582+0000","last_active":"2024-07-28T19:29:31.601878+0000","last_peered":"2024-07-28T19:29:31.601878+0000","last_clean":"2024-07-28T19:29:31.601878+0000","last_became_active":"2024-07-28T19:29:16.275714+0000","last_became_peered":"2024-07-28T19:29:16.275714+0000","last_unstale":"2024-07-28T19:29:31.601878+0000","last_undegraded":"2024-07-28T19:29:31.601878+0000","last_fullsized":"2024-07-28T19:29:31.601878+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:49:26.417025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.955571+0000","last_change":"2024-07-28T19:29:15.270203+0000","last_active":"2024-07-28T19:29:30.955571+0000","last_peered":"2024-07-28T19:29:30.955571+0000","last_clean":"2024-07-28T19:29:30.955571+0000","last_became_active":"2024-07-28T19:29:15.269343+0000","last_became_peered":"2024-07-28T19:29:15.269343+0000","last_unstale":"2024-07-28T19:29:30.955571+0000","last_undegraded":"2024-07-28T19:29:30.955571+0000","last_fullsized":"2024-07-28T19:29:30.955571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:01:57.007719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.948765+0000","last_change":"2024-07-28T19:29:16.446263+0000","last_active":"2024-07-28T19:29:30.948765+0000","last_peered":"2024-07-28T19:29:30.948765+0000","last_clean":"2024-07-28T19:29:30.948765+0000","last_became_active":"2024-07-28T19:29:16.445988+0000","last_became_peered":"2024-07-28T19:29:16.445988+0000","last_unstale":"2024-07-28T19:29:30.948765+0000","last_undegraded":"2024-07-28T19:29:30.948765+0000","last_fullsized":"2024-07-28T19:29:30.948765+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:52:15.881155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.963237+0000","last_change":"2024-07-28T19:29:15.270401+0000","last_active":"2024-07-28T19:29:30.963237+0000","last_peered":"2024-07-28T19:29:30.963237+0000","last_clean":"2024-07-28T19:29:30.963237+0000","last_became_active":"2024-07-28T19:29:15.268811+0000","last_became_peered":"2024-07-28T19:29:15.268811+0000","last_unstale":"2024-07-28T19:29:30.963237+0000","last_undegraded":"2024-07-28T19:29:30.963237+0000","last_fullsized":"2024-07-28T19:29:30.963237+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:45:42.090959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957923+0000","last_change":"2024-07-28T19:29:16.327119+0000","last_active":"2024-07-28T19:29:30.957923+0000","last_peered":"2024-07-28T19:29:30.957923+0000","last_clean":"2024-07-28T19:29:30.957923+0000","last_became_active":"2024-07-28T19:29:16.326690+0000","last_became_peered":"2024-07-28T19:29:16.326690+0000","last_unstale":"2024-07-28T19:29:30.957923+0000","last_undegraded":"2024-07-28T19:29:30.957923+0000","last_fullsized":"2024-07-28T19:29:30.957923+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:00:14.430162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958582+0000","last_change":"2024-07-28T19:29:15.248377+0000","last_active":"2024-07-28T19:29:30.958582+0000","last_peered":"2024-07-28T19:29:30.958582+0000","last_clean":"2024-07-28T19:29:30.958582+0000","last_became_active":"2024-07-28T19:29:15.247531+0000","last_became_peered":"2024-07-28T19:29:15.247531+0000","last_unstale":"2024-07-28T19:29:30.958582+0000","last_undegraded":"2024-07-28T19:29:30.958582+0000","last_fullsized":"2024-07-28T19:29:30.958582+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:24:12.524223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.956601+0000","last_change":"2024-07-28T19:29:15.297883+0000","last_active":"2024-07-28T19:29:30.956601+0000","last_peered":"2024-07-28T19:29:30.956601+0000","last_clean":"2024-07-28T19:29:30.956601+0000","last_became_active":"2024-07-28T19:29:15.297582+0000","last_became_peered":"2024-07-28T19:29:15.297582+0000","last_unstale":"2024-07-28T19:29:30.956601+0000","last_undegraded":"2024-07-28T19:29:30.956601+0000","last_fullsized":"2024-07-28T19:29:30.956601+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T06:18:22.320889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949495+0000","last_change":"2024-07-28T19:29:15.279264+0000","last_active":"2024-07-28T19:29:30.949495+0000","last_peered":"2024-07-28T19:29:30.949495+0000","last_clean":"2024-07-28T19:29:30.949495+0000","last_became_active":"2024-07-28T19:29:15.278728+0000","last_became_peered":"2024-07-28T19:29:15.278728+0000","last_unstale":"2024-07-28T19:29:30.949495+0000","last_undegraded":"2024-07-28T19:29:30.949495+0000","last_fullsized":"2024-07-28T19:29:30.949495+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:28:34.240386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957527+0000","last_change":"2024-07-28T19:29:15.307357+0000","last_active":"2024-07-28T19:29:30.957527+0000","last_peered":"2024-07-28T19:29:30.957527+0000","last_clean":"2024-07-28T19:29:30.957527+0000","last_became_active":"2024-07-28T19:29:15.306819+0000","last_became_peered":"2024-07-28T19:29:15.306819+0000","last_unstale":"2024-07-28T19:29:30.957527+0000","last_undegraded":"2024-07-28T19:29:30.957527+0000","last_fullsized":"2024-07-28T19:29:30.957527+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:50:24.186555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.951068+0000","last_change":"2024-07-28T19:29:15.302679+0000","last_active":"2024-07-28T19:29:30.951068+0000","last_peered":"2024-07-28T19:29:30.951068+0000","last_clean":"2024-07-28T19:29:30.951068+0000","last_became_active":"2024-07-28T19:29:15.302156+0000","last_became_peered":"2024-07-28T19:29:15.302156+0000","last_unstale":"2024-07-28T19:29:30.951068+0000","last_undegraded":"2024-07-28T19:29:30.951068+0000","last_fullsized":"2024-07-28T19:29:30.951068+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:54:06.863534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.958487+0000","last_change":"2024-07-28T19:29:15.600588+0000","last_active":"2024-07-28T19:29:30.958487+0000","last_peered":"2024-07-28T19:29:30.958487+0000","last_clean":"2024-07-28T19:29:30.958487+0000","last_became_active":"2024-07-28T19:29:15.599767+0000","last_became_peered":"2024-07-28T19:29:15.599767+0000","last_unstale":"2024-07-28T19:29:30.958487+0000","last_undegraded":"2024-07-28T19:29:30.958487+0000","last_fullsized":"2024-07-28T19:29:30.958487+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:09:05.954358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.952021+0000","last_change":"2024-07-28T19:29:15.292421+0000","last_active":"2024-07-28T19:29:30.952021+0000","last_peered":"2024-07-28T19:29:30.952021+0000","last_clean":"2024-07-28T19:29:30.952021+0000","last_became_active":"2024-07-28T19:29:15.291035+0000","last_became_peered":"2024-07-28T19:29:15.291035+0000","last_unstale":"2024-07-28T19:29:30.952021+0000","last_undegraded":"2024-07-28T19:29:30.952021+0000","last_fullsized":"2024-07-28T19:29:30.952021+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:51:06.527335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.956219+0000","last_change":"2024-07-28T19:29:16.446102+0000","last_active":"2024-07-28T19:29:30.956219+0000","last_peered":"2024-07-28T19:29:30.956219+0000","last_clean":"2024-07-28T19:29:30.956219+0000","last_became_active":"2024-07-28T19:29:16.445138+0000","last_became_peered":"2024-07-28T19:29:16.445138+0000","last_unstale":"2024-07-28T19:29:30.956219+0000","last_undegraded":"2024-07-28T19:29:30.956219+0000","last_fullsized":"2024-07-28T19:29:30.956219+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:35:18.586253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.957629+0000","last_change":"2024-07-28T19:29:15.281836+0000","last_active":"2024-07-28T19:29:30.957629+0000","last_peered":"2024-07-28T19:29:30.957629+0000","last_clean":"2024-07-28T19:29:30.957629+0000","last_became_active":"2024-07-28T19:29:15.281461+0000","last_became_peered":"2024-07-28T19:29:15.281461+0000","last_unstale":"2024-07-28T19:29:30.957629+0000","last_undegraded":"2024-07-28T19:29:30.957629+0000","last_fullsized":"2024-07-28T19:29:30.957629+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:57:35.879475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.956901+0000","last_change":"2024-07-28T19:29:15.304058+0000","last_active":"2024-07-28T19:29:30.956901+0000","last_peered":"2024-07-28T19:29:30.956901+0000","last_clean":"2024-07-28T19:29:30.956901+0000","last_became_active":"2024-07-28T19:29:15.303594+0000","last_became_peered":"2024-07-28T19:29:15.303594+0000","last_unstale":"2024-07-28T19:29:30.956901+0000","last_undegraded":"2024-07-28T19:29:30.956901+0000","last_fullsized":"2024-07-28T19:29:30.956901+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:22:07.859902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":22,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-07-28T19:29:30.949793+0000","last_change":"2024-07-28T19:29:15.270210+0000","last_active":"2024-07-28T19:29:30.949793+0000","last_peered":"2024-07-28T19:29:30.949793+0000","last_clean":"2024-07-28T19:29:30.949793+0000","last_became_active":"2024-07-28T19:29:15.268610+0000","last_became_peered":"2024-07-28T19:29:15.268610+0000","last_unstale":"2024-07-28T19:29:30.949793+0000","last_undegraded":"2024-07-28T19:29:30.949793+0000","last_fullsized":"2024-07-28T19:29:30.949793+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:07:50.326353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201349,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298680,"kb_used_data":1324,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446472,"statfs":{"total":95995035648,"available":95689187328,"internally_reserved":0,"allocated":1355776,"data_stored":915530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.85299999999999998,"15min":0.86899999999999999},"min":{"1min":0.52300000000000002,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.181,"5min":1.228,"15min":1.228},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.91000000000000003,"15min":0.91900000000000004},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.65500000000000003}]},{"osd":1,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.85299999999999998,"15min":0.85499999999999998},"min":{"1min":0.67000000000000004,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.2110000000000001,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.93400000000000005,"15min":0.92800000000000005},"min":{"1min":0.70599999999999996,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.78100000000000003}]},{"osd":2,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.97199999999999998,"15min":0.97799999999999998},"min":{"1min":0.62,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.163,"5min":1.284,"15min":1.284},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.97299999999999998,"15min":1.0009999999999999},"min":{"1min":0.63300000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2629999999999999,"5min":1.296,"15min":1.296},"last":1.006}]},{"osd":3,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.64100000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.1619999999999999,"5min":1.345,"15min":1.345},"last":0.84099999999999997},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.94999999999999996,"15min":0.96399999999999997},"min":{"1min":0.78300000000000003,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.93400000000000005}]},{"osd":4,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.88900000000000001,"15min":0.88300000000000001},"min":{"1min":0.623,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.087},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95599999999999996,"15min":1},"min":{"1min":0.60399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.2549999999999999,"5min":1.474,"15min":1.474},"last":0.752}]},{"osd":5,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.93200000000000005,"15min":0.95999999999999996},"min":{"1min":0.65100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.2170000000000001,"5min":1.272,"15min":1.272},"last":0.86899999999999999},{"interface":"front","average":{"1min":1.02,"5min":1.026,"15min":1.038},"min":{"1min":0.69399999999999995,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.331,"5min":1.335,"15min":1.335},"last":1.0329999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:30:44 2024","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0669999999999999,"15min":1.085},"min":{"1min":0.751,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.377,"5min":1.385,"15min":1.385},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.0609999999999999,"15min":1.093},"min":{"1min":0.746,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.4930000000000001,"5min":1.581,"15min":1.581},"last":1.056}]}]},{"osd":6,"up_from":47,"seq":201863462991,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":915442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:53 2024","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.83899999999999997,"15min":0.83699999999999997},"min":{"1min":0.54600000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.91100000000000003,"15min":0.89500000000000002},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51000000000000001},"max":{"1min":1.2649999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.77500000000000002}]},{"osd":1,"last update":"Sun Jul 28 19:30:53 2024","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.88600000000000001,"15min":0.85899999999999999},"min":{"1min":0.65800000000000003,"5min":0.503,"15min":0.503},"max":{"1min":1.179,"5min":3.2679999999999998,"15min":3.2679999999999998},"last":0.875},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.92500000000000004,"15min":0.872},"min":{"1min":0.76500000000000001,"5min":0.499,"15min":0.499},"max":{"1min":1.1639999999999999,"5min":1.359,"15min":1.359},"last":0.81899999999999995}]},{"osd":2,"last update":"Sun Jul 28 19:30:53 2024","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.91900000000000004,"15min":0.87},"min":{"1min":0.65700000000000003,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.97699999999999998},{"interface":"front","average":{"1min":1,"5min":0.95999999999999996,"15min":0.91200000000000003},"min":{"1min":0.66400000000000003,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.1919999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.94699999999999995}]},{"osd":3,"last update":"Sun Jul 28 19:30:53 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.97499999999999998,"15min":0.93700000000000006},"min":{"1min":0.79800000000000004,"5min":0.498,"15min":0.498},"max":{"1min":1.165,"5min":3.492,"15min":3.492},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95899999999999996,"15min":0.88500000000000001},"min":{"1min":0.77900000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.3300000000000001,"5min":1.427,"15min":1.427},"last":0.92400000000000004}]},{"osd":4,"last update":"Sun Jul 28 19:30:53 2024","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.88,"15min":0.82599999999999996},"min":{"1min":0.56499999999999995,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.2509999999999999,"5min":2.9380000000000002,"15min":2.9380000000000002},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.0049999999999999,"15min":0.86299999999999999},"min":{"1min":0.629,"5min":0.316,"15min":0.316},"max":{"1min":1.306,"5min":3.3919999999999999,"15min":3.3919999999999999},"last":1.2709999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:30:53 2024","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.98699999999999999,"15min":0.85699999999999998},"min":{"1min":0.67900000000000005,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.306,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.018,"5min":1.0349999999999999,"15min":0.874},"min":{"1min":0.63400000000000001,"5min":0.35199999999999998,"15min":0.33600000000000002},"max":{"1min":1.3140000000000001,"5min":3.665,"15min":3.665},"last":1.01}]},{"osd":7,"last update":"Sun Jul 28 19:30:38 2024","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.073,"15min":1.0629999999999999},"min":{"1min":0.78100000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.0489999999999999,"5min":3.0489999999999999,"15min":3.0489999999999999},"last":1.2390000000000001},{"interface":"front","average":{"1min":1.228,"5min":1.1160000000000001,"15min":1.101},"min":{"1min":0.77700000000000002,"5min":0.379,"15min":0.379},"max":{"1min":3.5790000000000002,"5min":3.5790000000000002,"15min":3.5790000000000002},"last":1.1970000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724633,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298080,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447072,"statfs":{"total":95995035648,"available":95689801728,"internally_reserved":0,"allocated":741376,"data_stored":324034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.91000000000000003,"15min":0.86799999999999999},"min":{"1min":0.58099999999999996,"5min":0.55000000000000004,"15min":0.50900000000000001},"max":{"1min":1.1990000000000001,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.874,"15min":0.84799999999999998},"min":{"1min":0.73099999999999998,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1100000000000001,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":1.159}]},{"osd":1,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.91700000000000004,"15min":0.873},"min":{"1min":0.59799999999999998,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.2769999999999999,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.90300000000000002,"15min":0.86899999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.077,"5min":1.347,"15min":1.347},"last":1.1919999999999999}]},{"osd":2,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.89800000000000002,"15min":0.871},"min":{"1min":0.55300000000000005,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.1779999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.93500000000000005,"15min":0.89700000000000002},"min":{"1min":0.60699999999999998,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.345,"5min":1.345,"15min":1.345},"last":1.2929999999999999}]},{"osd":3,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.93100000000000005,"15min":0.89700000000000002},"min":{"1min":0.66200000000000003,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1240000000000001,"5min":1.397,"15min":1.397},"last":1.117},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.97499999999999998,"15min":0.89900000000000002},"min":{"1min":0.71299999999999997,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.1830000000000001,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":1.0740000000000001}]},{"osd":4,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.83699999999999997,"15min":0.74399999999999999},"min":{"1min":0.63200000000000001,"5min":0.39900000000000002,"15min":0.377},"max":{"1min":0.98899999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.379},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.86099999999999999,"15min":0.79800000000000004},"min":{"1min":0.64900000000000002,"5min":0.377,"15min":0.35599999999999998},"max":{"1min":1.339,"5min":1.4870000000000001,"15min":1.4870000000000001},"last":1.3480000000000001}]},{"osd":6,"last update":"Sun Jul 28 19:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.97199999999999998,"15min":0.877},"min":{"1min":0.76000000000000001,"5min":0.373,"15min":0.35599999999999998},"max":{"1min":1.2490000000000001,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.997,"5min":0.98899999999999999,"15min":0.92700000000000005},"min":{"1min":0.66200000000000003,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.413,"5min":1.468,"15min":2.1669999999999998},"last":1.268}]},{"osd":7,"last update":"Sun Jul 28 19:30:46 2024","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.006,"15min":1},"min":{"1min":0.66200000000000003,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.2150000000000001,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":1.319},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.01,"15min":1.032},"min":{"1min":0.80100000000000005,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.343,"5min":1.516,"15min":1.516},"last":1.236}]}]},{"osd":4,"up_from":31,"seq":133143986274,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298092,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447060,"statfs":{"total":95995035648,"available":95689789440,"internally_reserved":0,"allocated":757760,"data_stored":325150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.88600000000000001,"15min":0.85399999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.85099999999999998,"15min":0.83199999999999996},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.2889999999999999,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.68500000000000005}]},{"osd":1,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.83799999999999997,"15min":0.80400000000000005},"min":{"1min":0.55200000000000005,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.042},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.89900000000000002,"15min":0.872},"min":{"1min":0.60299999999999998,"5min":0.57399999999999995,"15min":0.55600000000000005},"max":{"1min":1.145,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.72799999999999998}]},{"osd":2,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.91200000000000003,"15min":0.89300000000000002},"min":{"1min":0.67700000000000005,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.1930000000000001,"5min":1.298,"15min":1.298},"last":0.878},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.92000000000000004,"15min":0.875},"min":{"1min":0.56100000000000005,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.2250000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.78400000000000003}]},{"osd":3,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.94399999999999995,"15min":0.88800000000000001},"min":{"1min":0.78900000000000003,"5min":0.504,"15min":0.504},"max":{"1min":1.462,"5min":1.462,"15min":1.462},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.90000000000000002,"15min":0.86899999999999999},"min":{"1min":0.63100000000000001,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.2090000000000001,"5min":1.583,"15min":1.583},"last":1.115}]},{"osd":5,"last update":"Sun Jul 28 19:30:07 2024","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":0.93200000000000005,"15min":0.82599999999999996},"min":{"1min":0.58099999999999996,"5min":0.40200000000000002,"15min":0.39000000000000001},"max":{"1min":1.244,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.92500000000000004,"15min":0.83899999999999997},"min":{"1min":0.59399999999999997,"5min":0.33200000000000002,"15min":0.309},"max":{"1min":1.325,"5min":1.6919999999999999,"15min":1.6919999999999999},"last":0.83199999999999996}]},{"osd":6,"last update":"Sun Jul 28 19:30:54 2024","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":0.96999999999999997,"15min":0.92400000000000004},"min":{"1min":0.71399999999999997,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.3560000000000001,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.97299999999999998,"15min":0.875},"min":{"1min":0.65500000000000003,"5min":0.46400000000000002,"15min":0.36499999999999999},"max":{"1min":1.375,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.85599999999999998}]},{"osd":7,"last update":"Sun Jul 28 19:30:39 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.117,"15min":1.3580000000000001},"min":{"1min":0.77800000000000002,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.4239999999999999,"5min":8.7970000000000006,"15min":8.7970000000000006},"last":1.01},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.198,"15min":1.49},"min":{"1min":0.69299999999999995,"5min":0.378,"15min":0.378},"max":{"1min":1.423,"5min":10.920999999999999,"15min":10.920999999999999},"last":1.1779999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149804,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":741376,"data_stored":323807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.69299999999999995,"15min":0.68500000000000005},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.84099999999999997,"5min":1.909,"15min":1.909},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.79100000000000004,"15min":0.77400000000000002},"min":{"1min":0.36099999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.95599999999999996,"5min":1.556,"15min":1.556},"last":0.76000000000000001}]},{"osd":1,"last update":"Sun Jul 28 19:30:24 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.71499999999999997,"15min":0.70499999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.93400000000000005,"5min":1.583,"15min":1.583},"last":0.97299999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.69099999999999995,"15min":0.67300000000000004},"min":{"1min":0.39500000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.997,"5min":1.427,"15min":1.427},"last":0.85499999999999998}]},{"osd":2,"last update":"Sun Jul 28 19:30:24 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.79800000000000004,"15min":0.76400000000000001},"min":{"1min":0.45100000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.161,"5min":1.839,"15min":1.839},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.72299999999999998,"15min":0.67900000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.171,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.60899999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.90900000000000003,"15min":0.88},"min":{"1min":0.68600000000000005,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.1080000000000001,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.97099999999999997,"15min":1.052},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.1619999999999999,"5min":1.996,"15min":5.6539999999999999},"last":1.0640000000000001}]},{"osd":5,"last update":"Sun Jul 28 19:30:01 2024","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.96199999999999997,"15min":0.93100000000000005},"min":{"1min":0.75600000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":1.6040000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.96999999999999997,"15min":0.95899999999999996},"min":{"1min":0.746,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.208,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.0089999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:30:54 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.98899999999999999,"15min":1.0149999999999999},"min":{"1min":0.629,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.175,"5min":2.165,"15min":2.165},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.002,"15min":1.004},"min":{"1min":0.65000000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.2,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":1.089}]},{"osd":7,"last update":"Sun Jul 28 19:30:42 2024","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0389999999999999,"15min":1.077},"min":{"1min":0.66600000000000004,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2010000000000001,"5min":2.0880000000000001,"15min":2.0880000000000001},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.0409999999999999,"15min":1.071},"min":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.2090000000000001,"5min":1.704,"15min":1.704},"last":1.036}]}]},{"osd":2,"up_from":18,"seq":77309411449,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298080,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447072,"statfs":{"total":95995035648,"available":95689801728,"internally_reserved":0,"allocated":745472,"data_stored":324321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.61699999999999999,"15min":0.57899999999999996},"min":{"1min":0.39800000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":2.4569999999999999},"last":0.499},{"interface":"front","average":{"1min":0.82499999999999996,"5min":0.79400000000000004,"15min":0.70599999999999996},"min":{"1min":0.47099999999999997,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":1.3049999999999999,"5min":1.456,"15min":1.456},"last":0.59299999999999997}]},{"osd":1,"last update":"Sun Jul 28 19:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.68500000000000005,"15min":0.61299999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.30599999999999999},"max":{"1min":1.1779999999999999,"5min":1.252,"15min":1.252},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.745,"15min":0.67400000000000004},"min":{"1min":0.379,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":1.083,"5min":1.345,"15min":1.7350000000000001},"last":0.70699999999999996}]},{"osd":3,"last update":"Sun Jul 28 19:30:27 2024","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.80500000000000005,"15min":0.76300000000000001},"min":{"1min":0.41699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.272,"5min":1.3759999999999999,"15min":1.9790000000000001},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.84099999999999997,"15min":0.80500000000000005},"min":{"1min":0.46700000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.224,"5min":1.248,"15min":1.605},"last":0.81399999999999995}]},{"osd":4,"last update":"Sun Jul 28 19:30:18 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.88500000000000001,"15min":0.85899999999999999},"min":{"1min":0.67100000000000004,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":1.169},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.98699999999999999,"15min":0.93300000000000005},"min":{"1min":0.67100000000000004,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.427,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":0.95999999999999996}]},{"osd":5,"last update":"Sun Jul 28 19:30:10 2024","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.94199999999999995,"15min":1.0209999999999999},"min":{"1min":0.73499999999999999,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.498,"5min":1.498,"15min":3.9969999999999999},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.95599999999999996,"15min":1.1020000000000001},"min":{"1min":0.68200000000000005,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.228,"5min":1.504,"15min":5.3680000000000003},"last":0.93300000000000005}]},{"osd":6,"last update":"Sun Jul 28 19:30:54 2024","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.99099999999999999,"15min":1.02},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.3129999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.046},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.044,"15min":1.046},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.532,"5min":1.6859999999999999,"15min":1.6859999999999999},"last":0.90100000000000002}]},{"osd":7,"last update":"Sun Jul 28 19:30:46 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.069,"15min":1.091},"min":{"1min":0.64500000000000002,"5min":0.621,"15min":0.621},"max":{"1min":1.4299999999999999,"5min":1.766,"15min":1.766},"last":1.02},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0029999999999999,"15min":1.0289999999999999},"min":{"1min":0.60199999999999998,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.349,"5min":1.454,"15min":1.454},"last":1.1060000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574978,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":733184,"data_stored":323452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:30:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.65600000000000003,"15min":0.57599999999999996},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.84299999999999997,"5min":2.2530000000000001,"15min":2.2530000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.72599999999999998,"15min":0.63500000000000001},"min":{"1min":0.34000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.244,"5min":1.351,"15min":1.351},"last":0.46600000000000003}]},{"osd":2,"last update":"Sun Jul 28 19:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.68500000000000005,"15min":0.61699999999999999},"min":{"1min":0.442,"5min":0.28399999999999997,"15min":0.26400000000000001},"max":{"1min":1.0089999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.77200000000000002,"15min":0.71299999999999997},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.0469999999999999,"5min":1.4039999999999999,"15min":1.4039999999999999},"last":0.66900000000000004}]},{"osd":3,"last update":"Sun Jul 28 19:30:25 2024","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.83299999999999996,"15min":0.79700000000000004},"min":{"1min":0.36899999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.254,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.753,"15min":0.751},"min":{"1min":0.29899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.069,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.45300000000000001}]},{"osd":4,"last update":"Sun Jul 28 19:30:12 2024","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.008,"15min":0.93500000000000005},"min":{"1min":0.73099999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.3069999999999999,"5min":1.554,"15min":1.554},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.93500000000000005,"15min":0.90600000000000003},"min":{"1min":0.74199999999999999,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3779999999999999,"5min":1.53,"15min":1.53},"last":0.70199999999999996}]},{"osd":5,"last update":"Sun Jul 28 19:30:06 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.94599999999999995,"15min":0.91400000000000003},"min":{"1min":0.68799999999999994,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.381,"5min":1.605,"15min":1.605},"last":0.999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.024,"15min":0.997},"min":{"1min":0.78900000000000003,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.355,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.93100000000000005}]},{"osd":6,"last update":"Sun Jul 28 19:30:53 2024","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.087,"15min":1.089},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.6339999999999999,"5min":1.74,"15min":1.74},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.038,"15min":0.98999999999999999},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.4019999999999999,"5min":1.778,"15min":1.778},"last":0.95699999999999996}]},{"osd":7,"last update":"Sun Jul 28 19:30:50 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.0229999999999999,"15min":1.0109999999999999},"min":{"1min":0.629,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.323,"5min":1.677,"15min":1.677},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.018,"5min":1.0329999999999999,"15min":1.0780000000000001},"min":{"1min":0.70799999999999996,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.97299999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738508,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298660,"kb_used_data":1308,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446492,"statfs":{"total":95995035648,"available":95689207808,"internally_reserved":0,"allocated":1339392,"data_stored":914677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Jul 28 19:30:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.63500000000000001,"15min":0.57099999999999995},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.014,"5min":1.0569999999999999,"15min":1.127},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.70699999999999996,"15min":0.63400000000000001},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.93700000000000006,"5min":1.347,"15min":1.347},"last":0.78400000000000003}]},{"osd":2,"last update":"Sun Jul 28 19:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.74099999999999999,"15min":0.68100000000000005},"min":{"1min":0.35899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.996,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.70199999999999996,"15min":0.67400000000000004},"min":{"1min":0.34399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.96099999999999997,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.83099999999999996}]},{"osd":3,"last update":"Sun Jul 28 19:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.91200000000000003,"5min":1.292,"15min":1.292},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.80500000000000005,"15min":0.81799999999999995},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.318},"max":{"1min":1.0669999999999999,"5min":1.177,"15min":1.3220000000000001},"last":0.66600000000000004}]},{"osd":4,"last update":"Sun Jul 28 19:30:11 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.93300000000000005,"15min":0.85799999999999998},"min":{"1min":0.67600000000000005,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.2470000000000001,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.024},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.94999999999999996,"15min":0.90200000000000002},"min":{"1min":0.69599999999999995,"5min":0.437,"15min":0.437},"max":{"1min":1.1399999999999999,"5min":1.3,"15min":1.327},"last":0.878}]},{"osd":5,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.94099999999999995,"15min":0.93899999999999995},"min":{"1min":0.59299999999999997,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.2050000000000001,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.95099999999999996,"15min":0.92500000000000004},"min":{"1min":0.5,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.2310000000000001,"5min":1.385,"15min":1.385},"last":1.0029999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:30:00 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.012,"15min":1.024},"min":{"1min":0.53800000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":1.093},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0469999999999999,"15min":1.052},"min":{"1min":0.65100000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.5980000000000001,"5min":1.5980000000000001,"15min":1.5980000000000001},"last":1.113}]},{"osd":7,"last update":"Sun Jul 28 19:30:49 2024","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.0369999999999999,"15min":1.0229999999999999},"min":{"1min":0.60699999999999998,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":6.2489999999999997,"5min":6.2489999999999997,"15min":6.2489999999999997},"last":1.05},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.004,"15min":1.0349999999999999},"min":{"1min":0.55800000000000005,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.3520000000000001,"5min":1.573,"15min":1.573},"last":0.97999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-28T19:31:04.176 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-07-28T19:31:04.176 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-07-28T19:31:04.176 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-07-28T19:31:04.176 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:31:04.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:04 smithi100 bash[21828]: cluster 2024-07-28T19:31:02.910024+0000 mgr.y (mgr.14391) 176 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:04.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:04 smithi100 bash[21828]: audit 2024-07-28T19:31:03.525262+0000 mgr.y (mgr.14391) 177 : audit [DBG] from='client.24662 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:31:04.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:04 smithi100 bash[31715]: cluster 2024-07-28T19:31:02.910024+0000 mgr.y (mgr.14391) 176 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:04.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:04 smithi100 bash[31715]: audit 2024-07-28T19:31:03.525262+0000 mgr.y (mgr.14391) 177 : audit [DBG] from='client.24662 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:31:04.553 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:04 smithi138 bash[23952]: cluster 2024-07-28T19:31:02.910024+0000 mgr.y (mgr.14391) 176 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:04.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:04 smithi138 bash[23952]: audit 2024-07-28T19:31:03.525262+0000 mgr.y (mgr.14391) 177 : audit [DBG] from='client.24662 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:31:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:06 smithi100 bash[21828]: cluster 2024-07-28T19:31:04.910701+0000 mgr.y (mgr.14391) 178 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:06 smithi100 bash[31715]: cluster 2024-07-28T19:31:04.910701+0000 mgr.y (mgr.14391) 178 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:06.553 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:06 smithi138 bash[23952]: cluster 2024-07-28T19:31:04.910701+0000 mgr.y (mgr.14391) 178 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:06.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:31:06 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:31:08.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:08 smithi100 bash[21828]: cluster 2024-07-28T19:31:06.911852+0000 mgr.y (mgr.14391) 179 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:08.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:08 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:08] "GET /metrics HTTP/1.1" 200 239410 "" "Prometheus/2.43.0" 2024-07-28T19:31:08.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:08 smithi100 bash[31715]: cluster 2024-07-28T19:31:06.911852+0000 mgr.y (mgr.14391) 179 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:08.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:08 smithi138 bash[23952]: cluster 2024-07-28T19:31:06.911852+0000 mgr.y (mgr.14391) 179 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:08.967 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:31:10.351 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:10 smithi100 bash[21828]: cluster 2024-07-28T19:31:08.912485+0000 mgr.y (mgr.14391) 180 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:10.351 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:10 smithi100 bash[31715]: cluster 2024-07-28T19:31:08.912485+0000 mgr.y (mgr.14391) 180 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:10.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:10 smithi138 bash[23952]: cluster 2024-07-28T19:31:08.912485+0000 mgr.y (mgr.14391) 180 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:12.114 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:31:12.114 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-07-28T19:31:12.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:12 smithi100 bash[31715]: cluster 2024-07-28T19:31:10.913152+0000 mgr.y (mgr.14391) 181 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:12.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:12 smithi100 bash[31715]: audit 2024-07-28T19:31:12.119757+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.100:0/4184142358' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:31:12.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:12 smithi100 bash[21828]: cluster 2024-07-28T19:31:10.913152+0000 mgr.y (mgr.14391) 181 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:12.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:12 smithi100 bash[21828]: audit 2024-07-28T19:31:12.119757+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.100:0/4184142358' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:31:12.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:12 smithi138 bash[23952]: cluster 2024-07-28T19:31:10.913152+0000 mgr.y (mgr.14391) 181 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:12.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:12 smithi138 bash[23952]: audit 2024-07-28T19:31:12.119757+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.100:0/4184142358' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:31:12.835 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-07-28T19:31:12.836 INFO:teuthology.run_tasks:Running task print... 2024-07-28T19:31:12.847 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-07-28T19:31:12.847 INFO:teuthology.run_tasks:Running task parallel... 2024-07-28T19:31:12.999 INFO:teuthology.task.parallel:starting parallel... 2024-07-28T19:31:13.000 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-07-28T19:31:13.000 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-07-28T19:31:13.005 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-07-28T19:31:13.006 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-07-28T19:31:13.006 DEBUG:teuthology.orchestra.run.smithi100:> stat -- /home/ubuntu/cephtest/mnt.0 2024-07-28T19:31:13.006 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-07-28T19:31:13.007 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-07-28T19:31:13.007 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi100.front.sepia.ceph.com 2024-07-28T19:31:13.007 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-07-28T19:31:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-28T19:31:13.013 INFO:teuthology.orchestra.run.smithi100.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-07-28T19:31:13.014 DEBUG:teuthology.orchestra.run.smithi100:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-07-28T19:31:13.059 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-07-28T19:31:13.060 DEBUG:teuthology.orchestra.run.smithi100:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-07-28T19:31:13.107 INFO:tasks.workunit:timeout=3h 2024-07-28T19:31:13.107 INFO:tasks.workunit:cleanup=True 2024-07-28T19:31:13.107 DEBUG:teuthology.orchestra.run.smithi100:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-07-28T19:31:13.157 INFO:tasks.workunit.client.0.smithi100.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-07-28T19:31:14.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:14 smithi100 bash[21828]: cluster 2024-07-28T19:31:12.914183+0000 mgr.y (mgr.14391) 182 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:14.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:14 smithi100 bash[31715]: cluster 2024-07-28T19:31:12.914183+0000 mgr.y (mgr.14391) 182 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:14.553 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:14 smithi138 bash[23952]: cluster 2024-07-28T19:31:12.914183+0000 mgr.y (mgr.14391) 182 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:16.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:31:16 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:31:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:16 smithi138 bash[23952]: cluster 2024-07-28T19:31:14.914739+0000 mgr.y (mgr.14391) 183 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:16 smithi138 bash[23952]: audit 2024-07-28T19:31:15.910604+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:16.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:16 smithi100 bash[21828]: cluster 2024-07-28T19:31:14.914739+0000 mgr.y (mgr.14391) 183 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:16.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:16 smithi100 bash[21828]: audit 2024-07-28T19:31:15.910604+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:16.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:16 smithi100 bash[31715]: cluster 2024-07-28T19:31:14.914739+0000 mgr.y (mgr.14391) 183 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:16.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:16 smithi100 bash[31715]: audit 2024-07-28T19:31:15.910604+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:17.785 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:31:18.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:18 smithi100 bash[21828]: cluster 2024-07-28T19:31:16.915840+0000 mgr.y (mgr.14391) 184 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:18.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:18 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:18] "GET /metrics HTTP/1.1" 200 239409 "" "Prometheus/2.43.0" 2024-07-28T19:31:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:18 smithi100 bash[31715]: cluster 2024-07-28T19:31:16.915840+0000 mgr.y (mgr.14391) 184 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:18.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:18 smithi138 bash[23952]: cluster 2024-07-28T19:31:16.915840+0000 mgr.y (mgr.14391) 184 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:18.665 DEBUG:teuthology.orchestra.run.smithi100:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-07-28T19:31:18.712 INFO:tasks.workunit.client.0.smithi100.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-07-28T19:31:18.714 INFO:tasks.workunit.client.0.smithi100.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-07-28T19:31:18.714 INFO:tasks.workunit.client.0.smithi100.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-07-28T19:31:20.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:20 smithi138 bash[23952]: cluster 2024-07-28T19:31:18.916481+0000 mgr.y (mgr.14391) 185 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:20.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:20 smithi100 bash[31715]: cluster 2024-07-28T19:31:18.916481+0000 mgr.y (mgr.14391) 185 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:20.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:20 smithi100 bash[21828]: cluster 2024-07-28T19:31:18.916481+0000 mgr.y (mgr.14391) 185 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:22.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:22 smithi138 bash[23952]: cluster 2024-07-28T19:31:20.917108+0000 mgr.y (mgr.14391) 186 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:22.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:22 smithi100 bash[21828]: cluster 2024-07-28T19:31:20.917108+0000 mgr.y (mgr.14391) 186 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:22.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:22 smithi100 bash[31715]: cluster 2024-07-28T19:31:20.917108+0000 mgr.y (mgr.14391) 186 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:23.289 INFO:tasks.workunit.client.0.smithi100.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-07-28T19:31:23.424 INFO:tasks.workunit.client.0.smithi100.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-07-28T19:31:23.482 INFO:tasks.workunit.client.0.smithi100.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-07-28T19:31:23.484 INFO:tasks.workunit.client.0.smithi100.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-07-28T19:31:23.484 INFO:tasks.workunit.client.0.smithi100.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-07-28T19:31:23.529 INFO:tasks.workunit.client.0.smithi100.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-07-28T19:31:23.533 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-07-28T19:31:23.534 DEBUG:teuthology.orchestra.run.smithi100:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-07-28T19:31:23.579 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-07-28T19:31:23.580 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-07-28T19:31:23.580 DEBUG:teuthology.orchestra.run.smithi100:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-07-28T19:31:23.719 INFO:tasks.workunit.client.0.smithi100.stderr:warning: failed to enable lockdep 2024-07-28T19:31:23.719 INFO:tasks.workunit.client.0.smithi100.stdout:seed 68802 2024-07-28T19:31:23.719 INFO:tasks.workunit.client.0.smithi100.stdout:[==========] Running 406 tests from 15 test suites. 2024-07-28T19:31:23.719 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] Global test environment set-up. 2024-07-28T19:31:23.720 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 16 tests from TestJournalReplay 2024-07-28T19:31:24.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[22098]: [28/Jul/2024:19:31:24] ENGINE Bus STOPPING 2024-07-28T19:31:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[21828]: cluster 2024-07-28T19:31:22.918118+0000 mgr.y (mgr.14391) 187 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:24.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[21828]: audit 2024-07-28T19:31:24.184968+0000 mon.a (mon.0) 817 : audit [INF] from='client.? ' entity='client.admin' 2024-07-28T19:31:24.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[21828]: audit 2024-07-28T19:31:24.197318+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:31:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[31715]: cluster 2024-07-28T19:31:22.918118+0000 mgr.y (mgr.14391) 187 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[31715]: audit 2024-07-28T19:31:24.184968+0000 mon.a (mon.0) 817 : audit [INF] from='client.? ' entity='client.admin' 2024-07-28T19:31:24.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[31715]: audit 2024-07-28T19:31:24.197318+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:31:24.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:24 smithi138 bash[23952]: cluster 2024-07-28T19:31:22.918118+0000 mgr.y (mgr.14391) 187 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:24.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:24 smithi138 bash[23952]: audit 2024-07-28T19:31:24.184968+0000 mon.a (mon.0) 817 : audit [INF] from='client.? ' entity='client.admin' 2024-07-28T19:31:24.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:24 smithi138 bash[23952]: audit 2024-07-28T19:31:24.197318+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:31:24.969 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[22098]: [28/Jul/2024:19:31:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:31:24.969 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[22098]: [28/Jul/2024:19:31:24] ENGINE Bus STOPPED 2024-07-28T19:31:24.969 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[22098]: [28/Jul/2024:19:31:24] ENGINE Bus STARTING 2024-07-28T19:31:24.969 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[22098]: [28/Jul/2024:19:31:24] ENGINE Serving on http://:::9283 2024-07-28T19:31:24.969 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:24 smithi100 bash[22098]: [28/Jul/2024:19:31:24] ENGINE Bus STARTED 2024-07-28T19:31:24.970 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph config set global log_to_journald false --force' 2024-07-28T19:31:25.301 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-07-28T19:31:25.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:25 smithi138 bash[23952]: cluster 2024-07-28T19:31:24.268784+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-07-28T19:31:25.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:25 smithi138 bash[23952]: audit 2024-07-28T19:31:24.275749+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.100:0/1741865999' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:25.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:25 smithi138 bash[23952]: audit 2024-07-28T19:31:24.280090+0000 mon.a (mon.0) 819 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:25.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:25 smithi100 bash[21828]: cluster 2024-07-28T19:31:24.268784+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-07-28T19:31:25.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:25 smithi100 bash[21828]: audit 2024-07-28T19:31:24.275749+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.100:0/1741865999' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:25.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:25 smithi100 bash[21828]: audit 2024-07-28T19:31:24.280090+0000 mon.a (mon.0) 819 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:25 smithi100 bash[31715]: cluster 2024-07-28T19:31:24.268784+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-07-28T19:31:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:25 smithi100 bash[31715]: audit 2024-07-28T19:31:24.275749+0000 mon.c (mon.2) 131 : audit [INF] from='client.? 172.21.15.100:0/1741865999' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:25 smithi100 bash[31715]: audit 2024-07-28T19:31:24.280090+0000 mon.a (mon.0) 819 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:26.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:31:26 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:31:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:26 smithi138 bash[23952]: cluster 2024-07-28T19:31:24.918777+0000 mgr.y (mgr.14391) 188 : cluster [DBG] pgmap v163: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:26 smithi138 bash[23952]: cluster 2024-07-28T19:31:25.276758+0000 mon.a (mon.0) 820 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:26 smithi138 bash[23952]: audit 2024-07-28T19:31:25.284261+0000 mon.a (mon.0) 821 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:26 smithi138 bash[23952]: cluster 2024-07-28T19:31:25.284594+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-07-28T19:31:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:26 smithi100 bash[21828]: cluster 2024-07-28T19:31:24.918777+0000 mgr.y (mgr.14391) 188 : cluster [DBG] pgmap v163: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:26 smithi100 bash[21828]: cluster 2024-07-28T19:31:25.276758+0000 mon.a (mon.0) 820 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:26.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:26 smithi100 bash[21828]: audit 2024-07-28T19:31:25.284261+0000 mon.a (mon.0) 821 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:26.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:26 smithi100 bash[21828]: cluster 2024-07-28T19:31:25.284594+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-07-28T19:31:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:26 smithi100 bash[31715]: cluster 2024-07-28T19:31:24.918777+0000 mgr.y (mgr.14391) 188 : cluster [DBG] pgmap v163: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:26 smithi100 bash[31715]: cluster 2024-07-28T19:31:25.276758+0000 mon.a (mon.0) 820 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:26 smithi100 bash[31715]: audit 2024-07-28T19:31:25.284261+0000 mon.a (mon.0) 821 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:26.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:26 smithi100 bash[31715]: cluster 2024-07-28T19:31:25.284594+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-07-28T19:31:27.337 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.337 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.AioDiscardEvent (2036 ms) 2024-07-28T19:31:27.337 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-07-28T19:31:27.361 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.361 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.AioWriteEvent (25 ms) 2024-07-28T19:31:27.361 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-07-28T19:31:27.383 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.383 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.AioFlushEvent (22 ms) 2024-07-28T19:31:27.383 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-07-28T19:31:27.406 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.406 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.SnapCreate (23 ms) 2024-07-28T19:31:27.406 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-07-28T19:31:27.428 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.428 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.SnapProtect (21 ms) 2024-07-28T19:31:27.428 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-07-28T19:31:27.451 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.451 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.SnapUnprotect (23 ms) 2024-07-28T19:31:27.451 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-07-28T19:31:27.473 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.473 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.SnapRename (22 ms) 2024-07-28T19:31:27.473 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-07-28T19:31:27.493 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.493 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.SnapRollback (21 ms) 2024-07-28T19:31:27.493 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-07-28T19:31:27.513 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.514 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.SnapRemove (20 ms) 2024-07-28T19:31:27.514 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.Rename 2024-07-28T19:31:27.534 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.534 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.Rename (20 ms) 2024-07-28T19:31:27.534 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.Resize 2024-07-28T19:31:27.554 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.554 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.Resize (21 ms) 2024-07-28T19:31:27.554 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.Flatten 2024-07-28T19:31:27.574 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.574 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.Flatten (19 ms) 2024-07-28T19:31:27.574 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-07-28T19:31:27.594 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.594 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.UpdateFeatures (20 ms) 2024-07-28T19:31:27.594 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-07-28T19:31:27.613 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.614 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.MetadataSet (20 ms) 2024-07-28T19:31:27.614 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-07-28T19:31:27.633 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.633 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.MetadataRemove (20 ms) 2024-07-28T19:31:27.633 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-07-28T19:31:27.653 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:27.653 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalReplay.ObjectPosition (19 ms) 2024-07-28T19:31:27.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:27 smithi100 bash[21828]: cluster 2024-07-28T19:31:26.306469+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-07-28T19:31:27.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:27 smithi100 bash[31715]: cluster 2024-07-28T19:31:26.306469+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-07-28T19:31:27.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:27 smithi138 bash[23952]: cluster 2024-07-28T19:31:26.306469+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-07-28T19:31:28.330 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 16 tests from TestJournalReplay (2352 ms total) 2024-07-28T19:31:28.331 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:31:28.331 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 3 tests from TestJournalEntries 2024-07-28T19:31:28.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:28] "GET /metrics HTTP/1.1" 200 239409 "" "Prometheus/2.43.0" 2024-07-28T19:31:28.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[21828]: cluster 2024-07-28T19:31:26.919924+0000 mgr.y (mgr.14391) 189 : cluster [DBG] pgmap v166: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-07-28T19:31:28.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[21828]: cluster 2024-07-28T19:31:27.300971+0000 mon.a (mon.0) 824 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:28.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[21828]: cluster 2024-07-28T19:31:27.301019+0000 mon.a (mon.0) 825 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:28.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[21828]: cluster 2024-07-28T19:31:27.318151+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-07-28T19:31:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[31715]: cluster 2024-07-28T19:31:26.919924+0000 mgr.y (mgr.14391) 189 : cluster [DBG] pgmap v166: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-07-28T19:31:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[31715]: cluster 2024-07-28T19:31:27.300971+0000 mon.a (mon.0) 824 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[31715]: cluster 2024-07-28T19:31:27.301019+0000 mon.a (mon.0) 825 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:28 smithi100 bash[31715]: cluster 2024-07-28T19:31:27.318151+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-07-28T19:31:28.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:28 smithi138 bash[23952]: cluster 2024-07-28T19:31:26.919924+0000 mgr.y (mgr.14391) 189 : cluster [DBG] pgmap v166: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-07-28T19:31:28.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:28 smithi138 bash[23952]: cluster 2024-07-28T19:31:27.300971+0000 mon.a (mon.0) 824 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:28.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:28 smithi138 bash[23952]: cluster 2024-07-28T19:31:27.301019+0000 mon.a (mon.0) 825 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:28.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:28 smithi138 bash[23952]: cluster 2024-07-28T19:31:27.318151+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-07-28T19:31:28.988 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:31:29.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:29 smithi100 bash[21828]: cluster 2024-07-28T19:31:28.328345+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-07-28T19:31:29.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:29 smithi100 bash[31715]: cluster 2024-07-28T19:31:28.328345+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-07-28T19:31:29.803 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:29 smithi138 bash[23952]: cluster 2024-07-28T19:31:28.328345+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-07-28T19:31:30.326 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: cluster 2024-07-28T19:31:28.920483+0000 mgr.y (mgr.14391) 190 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: cluster 2024-07-28T19:31:29.332202+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: audit 2024-07-28T19:31:29.337781+0000 mon.a (mon.0) 829 : audit [INF] from='client.? 172.21.15.100:0/1196737158' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: audit 2024-07-28T19:31:29.615947+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: audit 2024-07-28T19:31:29.623915+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: audit 2024-07-28T19:31:29.831969+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: audit 2024-07-28T19:31:29.842219+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: audit 2024-07-28T19:31:30.170677+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: audit 2024-07-28T19:31:30.172148+0000 mon.c (mon.2) 133 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:31:30.539 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[21828]: audit 2024-07-28T19:31:30.182583+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: cluster 2024-07-28T19:31:28.920483+0000 mgr.y (mgr.14391) 190 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: cluster 2024-07-28T19:31:29.332202+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: audit 2024-07-28T19:31:29.337781+0000 mon.a (mon.0) 829 : audit [INF] from='client.? 172.21.15.100:0/1196737158' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: audit 2024-07-28T19:31:29.615947+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: audit 2024-07-28T19:31:29.623915+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: audit 2024-07-28T19:31:29.831969+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: audit 2024-07-28T19:31:29.842219+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: audit 2024-07-28T19:31:30.170677+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: audit 2024-07-28T19:31:30.172148+0000 mon.c (mon.2) 133 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:31:30.540 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:30 smithi100 bash[31715]: audit 2024-07-28T19:31:30.182583+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: cluster 2024-07-28T19:31:28.920483+0000 mgr.y (mgr.14391) 190 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: cluster 2024-07-28T19:31:29.332202+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: audit 2024-07-28T19:31:29.337781+0000 mon.a (mon.0) 829 : audit [INF] from='client.? 172.21.15.100:0/1196737158' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: audit 2024-07-28T19:31:29.615947+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: audit 2024-07-28T19:31:29.623915+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: audit 2024-07-28T19:31:29.831969+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: audit 2024-07-28T19:31:29.842219+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: audit 2024-07-28T19:31:30.170677+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: audit 2024-07-28T19:31:30.172148+0000 mon.c (mon.2) 133 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:31:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:30 smithi138 bash[23952]: audit 2024-07-28T19:31:30.182583+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:31.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:31 smithi100 bash[21828]: audit 2024-07-28T19:31:30.332397+0000 mon.a (mon.0) 835 : audit [INF] from='client.? 172.21.15.100:0/1196737158' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:31.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:31 smithi100 bash[21828]: cluster 2024-07-28T19:31:30.332462+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-28T19:31:31.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:31 smithi100 bash[21828]: audit 2024-07-28T19:31:30.910997+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:31.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:31 smithi100 bash[31715]: audit 2024-07-28T19:31:30.332397+0000 mon.a (mon.0) 835 : audit [INF] from='client.? 172.21.15.100:0/1196737158' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:31.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:31 smithi100 bash[31715]: cluster 2024-07-28T19:31:30.332462+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-28T19:31:31.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:31 smithi100 bash[31715]: audit 2024-07-28T19:31:30.910997+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:31.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:31 smithi138 bash[23952]: audit 2024-07-28T19:31:30.332397+0000 mon.a (mon.0) 835 : audit [INF] from='client.? 172.21.15.100:0/1196737158' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:31.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:31 smithi138 bash[23952]: cluster 2024-07-28T19:31:30.332462+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-07-28T19:31:31.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:31 smithi138 bash[23952]: audit 2024-07-28T19:31:30.910997+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:32.380 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:32.380 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalEntries.AioWrite (2053 ms) 2024-07-28T19:31:32.380 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-07-28T19:31:32.399 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:32.399 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalEntries.AioDiscard (20 ms) 2024-07-28T19:31:32.399 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-07-28T19:31:32.414 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:32.414 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestJournalEntries.AioFlush (14 ms) 2024-07-28T19:31:32.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:32 smithi100 bash[31715]: cluster 2024-07-28T19:31:30.921123+0000 mgr.y (mgr.14391) 191 : cluster [DBG] pgmap v172: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:32.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:32 smithi100 bash[31715]: cluster 2024-07-28T19:31:31.353625+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-07-28T19:31:32.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:32 smithi100 bash[21828]: cluster 2024-07-28T19:31:30.921123+0000 mgr.y (mgr.14391) 191 : cluster [DBG] pgmap v172: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:32.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:32 smithi100 bash[21828]: cluster 2024-07-28T19:31:31.353625+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-07-28T19:31:32.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:32 smithi138 bash[23952]: cluster 2024-07-28T19:31:30.921123+0000 mgr.y (mgr.14391) 191 : cluster [DBG] pgmap v172: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:32.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:32 smithi138 bash[23952]: cluster 2024-07-28T19:31:31.353625+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-07-28T19:31:33.176 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-07-28T19:31:33.369 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 3 tests from TestJournalEntries (2087 ms total) 2024-07-28T19:31:33.369 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:31:33.369 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 2 tests from TestTrash 2024-07-28T19:31:33.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:33 smithi100 bash[21828]: cluster 2024-07-28T19:31:32.351739+0000 mon.a (mon.0) 838 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-07-28T19:31:33.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:33 smithi100 bash[31715]: cluster 2024-07-28T19:31:32.351739+0000 mon.a (mon.0) 838 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-07-28T19:31:33.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:33 smithi138 bash[23952]: cluster 2024-07-28T19:31:32.351739+0000 mon.a (mon.0) 838 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-07-28T19:31:34.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:34 smithi100 bash[21828]: cluster 2024-07-28T19:31:32.921721+0000 mgr.y (mgr.14391) 192 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:34.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:34 smithi100 bash[21828]: cluster 2024-07-28T19:31:33.371847+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-07-28T19:31:34.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:34 smithi100 bash[31715]: cluster 2024-07-28T19:31:32.921721+0000 mgr.y (mgr.14391) 192 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:34.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:34 smithi100 bash[31715]: cluster 2024-07-28T19:31:33.371847+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-07-28T19:31:34.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:34 smithi138 bash[23952]: cluster 2024-07-28T19:31:32.921721+0000 mgr.y (mgr.14391) 192 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:34.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:34 smithi138 bash[23952]: cluster 2024-07-28T19:31:33.371847+0000 mon.a (mon.0) 839 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-07-28T19:31:35.395 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-07-28T19:31:35.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:35 smithi100 bash[21828]: cluster 2024-07-28T19:31:34.387536+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-07-28T19:31:35.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:35 smithi100 bash[21828]: audit 2024-07-28T19:31:34.393816+0000 mon.a (mon.0) 841 : audit [INF] from='client.? 172.21.15.100:0/3146039513' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:35.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:35 smithi100 bash[31715]: cluster 2024-07-28T19:31:34.387536+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-07-28T19:31:35.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:35 smithi100 bash[31715]: audit 2024-07-28T19:31:34.393816+0000 mon.a (mon.0) 841 : audit [INF] from='client.? 172.21.15.100:0/3146039513' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:35.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:35 smithi138 bash[23952]: cluster 2024-07-28T19:31:34.387536+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-07-28T19:31:35.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:35 smithi138 bash[23952]: audit 2024-07-28T19:31:34.393816+0000 mon.a (mon.0) 841 : audit [INF] from='client.? 172.21.15.100:0/3146039513' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:36.398 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:31:36 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:31:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:36 smithi100 bash[21828]: cluster 2024-07-28T19:31:34.922399+0000 mgr.y (mgr.14391) 193 : cluster [DBG] pgmap v178: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:36 smithi100 bash[21828]: cluster 2024-07-28T19:31:35.378085+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:36 smithi100 bash[21828]: audit 2024-07-28T19:31:35.399080+0000 mon.a (mon.0) 843 : audit [INF] from='client.? 172.21.15.100:0/3146039513' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:36.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:36 smithi100 bash[21828]: cluster 2024-07-28T19:31:35.399370+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-07-28T19:31:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:36 smithi100 bash[31715]: cluster 2024-07-28T19:31:34.922399+0000 mgr.y (mgr.14391) 193 : cluster [DBG] pgmap v178: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:36 smithi100 bash[31715]: cluster 2024-07-28T19:31:35.378085+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:36 smithi100 bash[31715]: audit 2024-07-28T19:31:35.399080+0000 mon.a (mon.0) 843 : audit [INF] from='client.? 172.21.15.100:0/3146039513' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:36 smithi100 bash[31715]: cluster 2024-07-28T19:31:35.399370+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-07-28T19:31:36.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:36 smithi138 bash[23952]: cluster 2024-07-28T19:31:34.922399+0000 mgr.y (mgr.14391) 193 : cluster [DBG] pgmap v178: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:36.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:36 smithi138 bash[23952]: cluster 2024-07-28T19:31:35.378085+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:36.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:36 smithi138 bash[23952]: audit 2024-07-28T19:31:35.399080+0000 mon.a (mon.0) 843 : audit [INF] from='client.? 172.21.15.100:0/3146039513' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:36.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:36 smithi138 bash[23952]: cluster 2024-07-28T19:31:35.399370+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-07-28T19:31:37.560 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestTrash.UserRemovingSource (2166 ms) 2024-07-28T19:31:37.561 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-07-28T19:31:37.638 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestTrash.RestoreMirroringSource (78 ms) 2024-07-28T19:31:37.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:37 smithi100 bash[21828]: cluster 2024-07-28T19:31:36.395975+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-07-28T19:31:37.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:37 smithi100 bash[31715]: cluster 2024-07-28T19:31:36.395975+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-07-28T19:31:37.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:37 smithi138 bash[23952]: cluster 2024-07-28T19:31:36.395975+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-07-28T19:31:37.977 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:31:38.399 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:38] "GET /metrics HTTP/1.1" 200 241732 "" "Prometheus/2.43.0" 2024-07-28T19:31:38.419 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 2 tests from TestTrash (2244 ms total) 2024-07-28T19:31:38.419 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:31:38.419 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 1 test from TestOperations 2024-07-28T19:31:38.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[31715]: cluster 2024-07-28T19:31:36.923557+0000 mgr.y (mgr.14391) 194 : cluster [DBG] pgmap v181: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:38.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[31715]: cluster 2024-07-28T19:31:37.401141+0000 mon.a (mon.0) 846 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:38.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[31715]: cluster 2024-07-28T19:31:37.401228+0000 mon.a (mon.0) 847 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:38.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[31715]: cluster 2024-07-28T19:31:37.414772+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-07-28T19:31:38.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[21828]: cluster 2024-07-28T19:31:36.923557+0000 mgr.y (mgr.14391) 194 : cluster [DBG] pgmap v181: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:38.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[21828]: cluster 2024-07-28T19:31:37.401141+0000 mon.a (mon.0) 846 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:38.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[21828]: cluster 2024-07-28T19:31:37.401228+0000 mon.a (mon.0) 847 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:38.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:38 smithi100 bash[21828]: cluster 2024-07-28T19:31:37.414772+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-07-28T19:31:38.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:38 smithi138 bash[23952]: cluster 2024-07-28T19:31:36.923557+0000 mgr.y (mgr.14391) 194 : cluster [DBG] pgmap v181: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:38.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:38 smithi138 bash[23952]: cluster 2024-07-28T19:31:37.401141+0000 mon.a (mon.0) 846 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:38.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:38 smithi138 bash[23952]: cluster 2024-07-28T19:31:37.401228+0000 mon.a (mon.0) 847 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:38.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:38 smithi138 bash[23952]: cluster 2024-07-28T19:31:37.414772+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-07-28T19:31:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:39 smithi100 bash[21828]: cluster 2024-07-28T19:31:38.417887+0000 mon.a (mon.0) 849 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-07-28T19:31:39.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:39 smithi100 bash[31715]: cluster 2024-07-28T19:31:38.417887+0000 mon.a (mon.0) 849 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-07-28T19:31:39.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:39 smithi138 bash[23952]: cluster 2024-07-28T19:31:38.417887+0000 mon.a (mon.0) 849 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-07-28T19:31:40.434 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-07-28T19:31:40.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:40 smithi100 bash[21828]: cluster 2024-07-28T19:31:38.923897+0000 mgr.y (mgr.14391) 195 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:40.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:40 smithi100 bash[21828]: cluster 2024-07-28T19:31:39.423738+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-07-28T19:31:40.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:40 smithi100 bash[21828]: audit 2024-07-28T19:31:39.424974+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.100:0/232779391' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:40.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:40 smithi100 bash[21828]: audit 2024-07-28T19:31:39.429919+0000 mon.a (mon.0) 851 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:40.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:40 smithi100 bash[31715]: cluster 2024-07-28T19:31:38.923897+0000 mgr.y (mgr.14391) 195 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:40.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:40 smithi100 bash[31715]: cluster 2024-07-28T19:31:39.423738+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-07-28T19:31:40.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:40 smithi100 bash[31715]: audit 2024-07-28T19:31:39.424974+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.100:0/232779391' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:40.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:40 smithi100 bash[31715]: audit 2024-07-28T19:31:39.429919+0000 mon.a (mon.0) 851 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:40 smithi138 bash[23952]: cluster 2024-07-28T19:31:38.923897+0000 mgr.y (mgr.14391) 195 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:40 smithi138 bash[23952]: cluster 2024-07-28T19:31:39.423738+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-07-28T19:31:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:40 smithi138 bash[23952]: audit 2024-07-28T19:31:39.424974+0000 mon.c (mon.2) 135 : audit [INF] from='client.? 172.21.15.100:0/232779391' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:40 smithi138 bash[23952]: audit 2024-07-28T19:31:39.429919+0000 mon.a (mon.0) 851 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:41.429 INFO:teuthology.orchestra.run.smithi100.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:31:41.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:41 smithi100 bash[21828]: audit 2024-07-28T19:31:40.436477+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:41.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:41 smithi100 bash[21828]: cluster 2024-07-28T19:31:40.436862+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-07-28T19:31:41.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:41 smithi100 bash[31715]: audit 2024-07-28T19:31:40.436477+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:41.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:41 smithi100 bash[31715]: cluster 2024-07-28T19:31:40.436862+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-07-28T19:31:41.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:41 smithi138 bash[23952]: audit 2024-07-28T19:31:40.436477+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:41.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:41 smithi138 bash[23952]: cluster 2024-07-28T19:31:40.436862+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-07-28T19:31:42.187 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch ps' 2024-07-28T19:31:42.476 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:31:42.476 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestOperations.DisableJournalingCorrupt (2042 ms) 2024-07-28T19:31:42.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: cluster 2024-07-28T19:31:40.924449+0000 mgr.y (mgr.14391) 196 : cluster [DBG] pgmap v187: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: audit 2024-07-28T19:31:41.424529+0000 mgr.y (mgr.14391) 197 : audit [DBG] from='client.14823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: cephadm 2024-07-28T19:31:41.425849+0000 mgr.y (mgr.14391) 198 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: audit 2024-07-28T19:31:41.434179+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: audit 2024-07-28T19:31:41.435465+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: cluster 2024-07-28T19:31:41.448419+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: audit 2024-07-28T19:31:41.451899+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: audit 2024-07-28T19:31:41.453237+0000 mon.c (mon.2) 138 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: audit 2024-07-28T19:31:41.460172+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[21828]: cephadm 2024-07-28T19:31:41.495425+0000 mgr.y (mgr.14391) 199 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: cluster 2024-07-28T19:31:40.924449+0000 mgr.y (mgr.14391) 196 : cluster [DBG] pgmap v187: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: audit 2024-07-28T19:31:41.424529+0000 mgr.y (mgr.14391) 197 : audit [DBG] from='client.14823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: cephadm 2024-07-28T19:31:41.425849+0000 mgr.y (mgr.14391) 198 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: audit 2024-07-28T19:31:41.434179+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: audit 2024-07-28T19:31:41.435465+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: cluster 2024-07-28T19:31:41.448419+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: audit 2024-07-28T19:31:41.451899+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: audit 2024-07-28T19:31:41.453237+0000 mon.c (mon.2) 138 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: audit 2024-07-28T19:31:41.460172+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:42 smithi100 bash[31715]: cephadm 2024-07-28T19:31:41.495425+0000 mgr.y (mgr.14391) 199 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:31:42.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: cluster 2024-07-28T19:31:40.924449+0000 mgr.y (mgr.14391) 196 : cluster [DBG] pgmap v187: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:42.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: audit 2024-07-28T19:31:41.424529+0000 mgr.y (mgr.14391) 197 : audit [DBG] from='client.14823 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:31:42.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: cephadm 2024-07-28T19:31:41.425849+0000 mgr.y (mgr.14391) 198 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:31:42.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: audit 2024-07-28T19:31:41.434179+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:42.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: audit 2024-07-28T19:31:41.435465+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:31:42.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: cluster 2024-07-28T19:31:41.448419+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-07-28T19:31:42.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: audit 2024-07-28T19:31:41.451899+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:31:42.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: audit 2024-07-28T19:31:41.453237+0000 mon.c (mon.2) 138 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:31:42.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: audit 2024-07-28T19:31:41.460172+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:42.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:42 smithi138 bash[23952]: cephadm 2024-07-28T19:31:41.495425+0000 mgr.y (mgr.14391) 199 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 2024-07-28T19:31:43.468 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 1 test from TestOperations (2042 ms total) 2024-07-28T19:31:43.490 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:31:43.490 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 4 tests from TestObjectMap 2024-07-28T19:31:43.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:43 smithi100 bash[21828]: cluster 2024-07-28T19:31:42.447391+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-07-28T19:31:43.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:43 smithi100 bash[31715]: cluster 2024-07-28T19:31:42.447391+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-07-28T19:31:43.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:43 smithi138 bash[23952]: cluster 2024-07-28T19:31:42.447391+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-07-28T19:31:44.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:44 smithi100 bash[21828]: cluster 2024-07-28T19:31:42.925160+0000 mgr.y (mgr.14391) 200 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:44.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:44 smithi100 bash[21828]: cluster 2024-07-28T19:31:43.471593+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-07-28T19:31:44.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:44 smithi100 bash[31715]: cluster 2024-07-28T19:31:42.925160+0000 mgr.y (mgr.14391) 200 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:44.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:44 smithi100 bash[31715]: cluster 2024-07-28T19:31:43.471593+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-07-28T19:31:44.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:44 smithi138 bash[23952]: cluster 2024-07-28T19:31:42.925160+0000 mgr.y (mgr.14391) 200 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:44.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:44 smithi138 bash[23952]: cluster 2024-07-28T19:31:43.471593+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-07-28T19:31:45.471 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-07-28T19:31:45.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:45 smithi138 bash[23952]: cluster 2024-07-28T19:31:44.479903+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-07-28T19:31:45.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:45 smithi138 bash[23952]: audit 2024-07-28T19:31:44.483298+0000 mon.c (mon.2) 139 : audit [INF] from='client.? 172.21.15.100:0/3798126414' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:45.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:45 smithi138 bash[23952]: audit 2024-07-28T19:31:44.492593+0000 mon.a (mon.0) 860 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:45.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:45 smithi138 bash[23952]: audit 2024-07-28T19:31:45.474859+0000 mon.a (mon.0) 861 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:45.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:45 smithi138 bash[23952]: cluster 2024-07-28T19:31:45.475159+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-07-28T19:31:45.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[21828]: cluster 2024-07-28T19:31:44.479903+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-07-28T19:31:45.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[21828]: audit 2024-07-28T19:31:44.483298+0000 mon.c (mon.2) 139 : audit [INF] from='client.? 172.21.15.100:0/3798126414' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:45.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[21828]: audit 2024-07-28T19:31:44.492593+0000 mon.a (mon.0) 860 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:45.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[21828]: audit 2024-07-28T19:31:45.474859+0000 mon.a (mon.0) 861 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:45.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[21828]: cluster 2024-07-28T19:31:45.475159+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-07-28T19:31:45.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[31715]: cluster 2024-07-28T19:31:44.479903+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-07-28T19:31:45.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[31715]: audit 2024-07-28T19:31:44.483298+0000 mon.c (mon.2) 139 : audit [INF] from='client.? 172.21.15.100:0/3798126414' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:45.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[31715]: audit 2024-07-28T19:31:44.492593+0000 mon.a (mon.0) 860 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:45.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[31715]: audit 2024-07-28T19:31:45.474859+0000 mon.a (mon.0) 861 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:45.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:45 smithi100 bash[31715]: cluster 2024-07-28T19:31:45.475159+0000 mon.a (mon.0) 862 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-07-28T19:31:46.502 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:31:46 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:31:46.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:46 smithi138 bash[23952]: cluster 2024-07-28T19:31:44.925829+0000 mgr.y (mgr.14391) 201 : cluster [DBG] pgmap v193: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:46.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:46 smithi138 bash[23952]: cluster 2024-07-28T19:31:45.483883+0000 mon.a (mon.0) 863 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:46.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:46 smithi138 bash[23952]: audit 2024-07-28T19:31:45.919905+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:46.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:46 smithi138 bash[23952]: audit 2024-07-28T19:31:45.922437+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:46.975 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:31:46.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:46 smithi100 bash[21828]: cluster 2024-07-28T19:31:44.925829+0000 mgr.y (mgr.14391) 201 : cluster [DBG] pgmap v193: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:46.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:46 smithi100 bash[21828]: cluster 2024-07-28T19:31:45.483883+0000 mon.a (mon.0) 863 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:46.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:46 smithi100 bash[21828]: audit 2024-07-28T19:31:45.919905+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:46.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:46 smithi100 bash[21828]: audit 2024-07-28T19:31:45.922437+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:46.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:46 smithi100 bash[31715]: cluster 2024-07-28T19:31:44.925829+0000 mgr.y (mgr.14391) 201 : cluster [DBG] pgmap v193: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:46.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:46 smithi100 bash[31715]: cluster 2024-07-28T19:31:45.483883+0000 mon.a (mon.0) 863 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:46.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:46 smithi100 bash[31715]: audit 2024-07-28T19:31:45.919905+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:31:46.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:46 smithi100 bash[31715]: audit 2024-07-28T19:31:45.922437+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:31:47.596 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2126 ms) 2024-07-28T19:31:47.597 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-07-28T19:31:47.649 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (52 ms) 2024-07-28T19:31:47.649 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-07-28T19:31:47.722 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (74 ms) 2024-07-28T19:31:47.722 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-07-28T19:31:47.782 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (59 ms) 2024-07-28T19:31:47.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:47 smithi138 bash[23952]: cluster 2024-07-28T19:31:46.519903+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-07-28T19:31:47.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:47 smithi100 bash[21828]: cluster 2024-07-28T19:31:46.519903+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-07-28T19:31:47.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:47 smithi100 bash[31715]: cluster 2024-07-28T19:31:46.519903+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-07-28T19:31:48.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:48] "GET /metrics HTTP/1.1" 200 242176 "" "Prometheus/2.43.0" 2024-07-28T19:31:48.542 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 4 tests from TestObjectMap (2311 ms total) 2024-07-28T19:31:48.542 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:31:48.542 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 2 tests from TestMirroringWatcher 2024-07-28T19:31:48.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:48 smithi138 bash[23952]: cluster 2024-07-28T19:31:46.927004+0000 mgr.y (mgr.14391) 202 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:48.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:48 smithi138 bash[23952]: cluster 2024-07-28T19:31:47.515107+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:48.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:48 smithi138 bash[23952]: cluster 2024-07-28T19:31:47.515170+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:48.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:48 smithi138 bash[23952]: cluster 2024-07-28T19:31:47.531623+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-07-28T19:31:48.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[21828]: cluster 2024-07-28T19:31:46.927004+0000 mgr.y (mgr.14391) 202 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:48.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[21828]: cluster 2024-07-28T19:31:47.515107+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:48.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[21828]: cluster 2024-07-28T19:31:47.515170+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:48.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[21828]: cluster 2024-07-28T19:31:47.531623+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-07-28T19:31:48.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[31715]: cluster 2024-07-28T19:31:46.927004+0000 mgr.y (mgr.14391) 202 : cluster [DBG] pgmap v196: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:48.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[31715]: cluster 2024-07-28T19:31:47.515107+0000 mon.a (mon.0) 866 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:31:48.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[31715]: cluster 2024-07-28T19:31:47.515170+0000 mon.a (mon.0) 867 : cluster [INF] Cluster is now healthy 2024-07-28T19:31:48.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:48 smithi100 bash[31715]: cluster 2024-07-28T19:31:47.531623+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-07-28T19:31:49.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:49 smithi138 bash[23952]: cluster 2024-07-28T19:31:48.542002+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-07-28T19:31:49.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:49 smithi100 bash[21828]: cluster 2024-07-28T19:31:48.542002+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-07-28T19:31:49.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:49 smithi100 bash[31715]: cluster 2024-07-28T19:31:48.542002+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-07-28T19:31:50.555 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-07-28T19:31:50.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:50 smithi100 bash[31715]: cluster 2024-07-28T19:31:48.927708+0000 mgr.y (mgr.14391) 203 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:50.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:50 smithi100 bash[31715]: cluster 2024-07-28T19:31:49.555460+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-07-28T19:31:50.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:50 smithi100 bash[31715]: audit 2024-07-28T19:31:49.558800+0000 mon.a (mon.0) 871 : audit [INF] from='client.? 172.21.15.100:0/2241522715' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:50 smithi100 bash[21828]: cluster 2024-07-28T19:31:48.927708+0000 mgr.y (mgr.14391) 203 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:50 smithi100 bash[21828]: cluster 2024-07-28T19:31:49.555460+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-07-28T19:31:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:50 smithi100 bash[21828]: audit 2024-07-28T19:31:49.558800+0000 mon.a (mon.0) 871 : audit [INF] from='client.? 172.21.15.100:0/2241522715' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:51.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:50 smithi138 bash[23952]: cluster 2024-07-28T19:31:48.927708+0000 mgr.y (mgr.14391) 203 : cluster [DBG] pgmap v199: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:51.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:50 smithi138 bash[23952]: cluster 2024-07-28T19:31:49.555460+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-07-28T19:31:51.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:50 smithi138 bash[23952]: audit 2024-07-28T19:31:49.558800+0000 mon.a (mon.0) 871 : audit [INF] from='client.? 172.21.15.100:0/2241522715' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:51.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:51 smithi100 bash[31715]: audit 2024-07-28T19:31:50.557798+0000 mon.a (mon.0) 872 : audit [INF] from='client.? 172.21.15.100:0/2241522715' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:51.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:51 smithi100 bash[31715]: cluster 2024-07-28T19:31:50.558341+0000 mon.a (mon.0) 873 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-07-28T19:31:51.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:51 smithi100 bash[21828]: audit 2024-07-28T19:31:50.557798+0000 mon.a (mon.0) 872 : audit [INF] from='client.? 172.21.15.100:0/2241522715' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:51.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:51 smithi100 bash[21828]: cluster 2024-07-28T19:31:50.558341+0000 mon.a (mon.0) 873 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-07-28T19:31:52.006 INFO:teuthology.orchestra.run.smithi100.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-28T19:31:52.006 INFO:teuthology.orchestra.run.smithi100.stdout:alertmanager.a smithi100 *:9093,9094 running (4m) 22s ago 5m 13.8M - 0.25.0 c8568f914cd2 371ac411af1c 2024-07-28T19:31:52.006 INFO:teuthology.orchestra.run.smithi100.stdout:grafana.a smithi138 *:3000 running (4m) 22s ago 4m 79.1M - 9.4.7 954c08fa6188 e6e5a9627320 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi100.ppcqpk smithi100 running (2m) 22s ago 2m 11.5M - 17.2.7-1155-g3120d060 5ec745195f7e 24cab9a6a84c 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi138.nphkis smithi138 running (2m) 22s ago 2m 13.1M - 17.2.7-1155-g3120d060 5ec745195f7e cc36fa0642f5 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.x smithi138 *:8443 running (13m) 22s ago 13m 388M - 17.2.7-1155-g3120d060 5ec745195f7e 262bf807430e 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.y smithi100 *:9283 running (15m) 22s ago 15m 440M - 17.2.7-1155-g3120d060 5ec745195f7e 70016d759fcc 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:mon.a smithi100 running (15m) 22s ago 15m 46.9M 2048M 17.2.7-1155-g3120d060 5ec745195f7e c0ba6296a8a2 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:mon.b smithi138 running (13m) 22s ago 13m 36.9M 2048M 17.2.7-1155-g3120d060 5ec745195f7e a76f97e1f96b 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:mon.c smithi100 running (13m) 22s ago 13m 38.9M 2048M 17.2.7-1155-g3120d060 5ec745195f7e 59bb2850a096 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.a smithi100 *:9100 running (5m) 22s ago 5m 8192k - 1.5.0 0da6a335fe13 f51aa4fe5398 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.b smithi138 *:9100 running (5m) 22s ago 5m 8552k - 1.5.0 0da6a335fe13 7483ac96ecf1 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:osd.0 smithi100 running (12m) 22s ago 12m 46.7M 4060M 17.2.7-1155-g3120d060 5ec745195f7e bc95b51f52b0 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:osd.1 smithi100 running (11m) 22s ago 11m 45.0M 4060M 17.2.7-1155-g3120d060 5ec745195f7e ceb5589accd5 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:osd.2 smithi100 running (10m) 22s ago 10m 43.5M 4060M 17.2.7-1155-g3120d060 5ec745195f7e 1b4df5053189 2024-07-28T19:31:52.007 INFO:teuthology.orchestra.run.smithi100.stdout:osd.3 smithi100 running (9m) 22s ago 9m 42.2M 4060M 17.2.7-1155-g3120d060 5ec745195f7e cf8d59f38b6d 2024-07-28T19:31:52.008 INFO:teuthology.orchestra.run.smithi100.stdout:osd.4 smithi138 running (8m) 22s ago 9m 43.9M 4060M 17.2.7-1155-g3120d060 5ec745195f7e feff0c636bab 2024-07-28T19:31:52.008 INFO:teuthology.orchestra.run.smithi100.stdout:osd.5 smithi138 running (8m) 22s ago 8m 43.2M 4060M 17.2.7-1155-g3120d060 5ec745195f7e 33bea4a7c9ec 2024-07-28T19:31:52.008 INFO:teuthology.orchestra.run.smithi100.stdout:osd.6 smithi138 running (7m) 22s ago 7m 41.6M 4060M 17.2.7-1155-g3120d060 5ec745195f7e 11e0eaf60c0b 2024-07-28T19:31:52.008 INFO:teuthology.orchestra.run.smithi100.stdout:osd.7 smithi138 running (6m) 22s ago 6m 44.6M 4060M 17.2.7-1155-g3120d060 5ec745195f7e fd79c6c44f0f 2024-07-28T19:31:52.008 INFO:teuthology.orchestra.run.smithi100.stdout:prometheus.a smithi138 *:9095 running (4m) 22s ago 5m 49.6M - 2.43.0 a07b618ecd1d bec45d0b5c04 2024-07-28T19:31:52.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:51 smithi138 bash[23952]: audit 2024-07-28T19:31:50.557798+0000 mon.a (mon.0) 872 : audit [INF] from='client.? 172.21.15.100:0/2241522715' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:52.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:51 smithi138 bash[23952]: cluster 2024-07-28T19:31:50.558341+0000 mon.a (mon.0) 873 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-07-28T19:31:52.645 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2089 ms) 2024-07-28T19:31:52.645 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-07-28T19:31:52.687 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (43 ms) 2024-07-28T19:31:52.954 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:52 smithi100 bash[31715]: cluster 2024-07-28T19:31:50.928164+0000 mgr.y (mgr.14391) 204 : cluster [DBG] pgmap v202: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:52.954 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:52 smithi100 bash[31715]: cluster 2024-07-28T19:31:51.591538+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-07-28T19:31:52.954 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:52 smithi100 bash[21828]: cluster 2024-07-28T19:31:50.928164+0000 mgr.y (mgr.14391) 204 : cluster [DBG] pgmap v202: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:52.954 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:52 smithi100 bash[21828]: cluster 2024-07-28T19:31:51.591538+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-07-28T19:31:52.956 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'echo wait for minority of mons to upgrade' 2024-07-28T19:31:53.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:52 smithi138 bash[23952]: cluster 2024-07-28T19:31:50.928164+0000 mgr.y (mgr.14391) 204 : cluster [DBG] pgmap v202: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:53.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:52 smithi138 bash[23952]: cluster 2024-07-28T19:31:51.591538+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-07-28T19:31:53.603 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 2 tests from TestMirroringWatcher (2132 ms total) 2024-07-28T19:31:53.603 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:31:53.603 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 57 tests from TestMigration 2024-07-28T19:31:53.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:53 smithi100 bash[31715]: audit 2024-07-28T19:31:51.998761+0000 mgr.y (mgr.14391) 205 : audit [DBG] from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:31:53.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:53 smithi100 bash[31715]: cluster 2024-07-28T19:31:52.596497+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-07-28T19:31:53.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:53 smithi100 bash[21828]: audit 2024-07-28T19:31:51.998761+0000 mgr.y (mgr.14391) 205 : audit [DBG] from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:31:53.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:53 smithi100 bash[21828]: cluster 2024-07-28T19:31:52.596497+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-07-28T19:31:54.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:53 smithi138 bash[23952]: audit 2024-07-28T19:31:51.998761+0000 mgr.y (mgr.14391) 205 : audit [DBG] from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:31:54.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:53 smithi138 bash[23952]: cluster 2024-07-28T19:31:52.596497+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-07-28T19:31:54.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:54 smithi100 bash[31715]: cluster 2024-07-28T19:31:52.928608+0000 mgr.y (mgr.14391) 206 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:54.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:54 smithi100 bash[31715]: cluster 2024-07-28T19:31:53.606899+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-07-28T19:31:54.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:54 smithi100 bash[21828]: cluster 2024-07-28T19:31:52.928608+0000 mgr.y (mgr.14391) 206 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:54.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:54 smithi100 bash[21828]: cluster 2024-07-28T19:31:53.606899+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-07-28T19:31:55.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:54 smithi138 bash[23952]: cluster 2024-07-28T19:31:52.928608+0000 mgr.y (mgr.14391) 206 : cluster [DBG] pgmap v205: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:31:55.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:54 smithi138 bash[23952]: cluster 2024-07-28T19:31:53.606899+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-07-28T19:31:55.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:55 smithi100 bash[31715]: cluster 2024-07-28T19:31:54.611497+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-07-28T19:31:55.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:55 smithi100 bash[31715]: audit 2024-07-28T19:31:54.613118+0000 mon.c (mon.2) 141 : audit [INF] from='client.? 172.21.15.100:0/728099950' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:55.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:55 smithi100 bash[31715]: audit 2024-07-28T19:31:54.614280+0000 mon.a (mon.0) 878 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:55 smithi100 bash[21828]: cluster 2024-07-28T19:31:54.611497+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-07-28T19:31:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:55 smithi100 bash[21828]: audit 2024-07-28T19:31:54.613118+0000 mon.c (mon.2) 141 : audit [INF] from='client.? 172.21.15.100:0/728099950' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:55 smithi100 bash[21828]: audit 2024-07-28T19:31:54.614280+0000 mon.a (mon.0) 878 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:56.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:55 smithi138 bash[23952]: cluster 2024-07-28T19:31:54.611497+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-07-28T19:31:56.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:55 smithi138 bash[23952]: audit 2024-07-28T19:31:54.613118+0000 mon.c (mon.2) 141 : audit [INF] from='client.? 172.21.15.100:0/728099950' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:56.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:55 smithi138 bash[23952]: audit 2024-07-28T19:31:54.614280+0000 mon.a (mon.0) 878 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:31:56.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:31:56 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:31:56.634 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.Empty 2024-07-28T19:31:56.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:56 smithi100 bash[21828]: cluster 2024-07-28T19:31:54.929278+0000 mgr.y (mgr.14391) 207 : cluster [DBG] pgmap v208: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:56.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:56 smithi100 bash[21828]: cluster 2024-07-28T19:31:55.608144+0000 mon.a (mon.0) 879 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:56.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:56 smithi100 bash[21828]: audit 2024-07-28T19:31:55.630479+0000 mon.a (mon.0) 880 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:56.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:56 smithi100 bash[21828]: cluster 2024-07-28T19:31:55.630946+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-07-28T19:31:56.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:56 smithi100 bash[31715]: cluster 2024-07-28T19:31:54.929278+0000 mgr.y (mgr.14391) 207 : cluster [DBG] pgmap v208: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:56.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:56 smithi100 bash[31715]: cluster 2024-07-28T19:31:55.608144+0000 mon.a (mon.0) 879 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:56.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:56 smithi100 bash[31715]: audit 2024-07-28T19:31:55.630479+0000 mon.a (mon.0) 880 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:56.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:56 smithi100 bash[31715]: cluster 2024-07-28T19:31:55.630946+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-07-28T19:31:57.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:56 smithi138 bash[23952]: cluster 2024-07-28T19:31:54.929278+0000 mgr.y (mgr.14391) 207 : cluster [DBG] pgmap v208: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:57.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:56 smithi138 bash[23952]: cluster 2024-07-28T19:31:55.608144+0000 mon.a (mon.0) 879 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:31:57.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:56 smithi138 bash[23952]: audit 2024-07-28T19:31:55.630479+0000 mon.a (mon.0) 880 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:31:57.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:56 smithi138 bash[23952]: cluster 2024-07-28T19:31:55.630946+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-07-28T19:31:57.742 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:31:57.963 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:57 smithi100 bash[21828]: cluster 2024-07-28T19:31:56.636636+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-07-28T19:31:57.963 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:57 smithi100 bash[31715]: cluster 2024-07-28T19:31:56.636636+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-07-28T19:31:58.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:57 smithi138 bash[23952]: cluster 2024-07-28T19:31:56.636636+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-07-28T19:31:58.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:31:58 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:31:58] "GET /metrics HTTP/1.1" 200 242176 "" "Prometheus/2.43.0" 2024-07-28T19:31:58.736 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:31:58.758 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:31:58.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:58 smithi100 bash[21828]: cluster 2024-07-28T19:31:56.930853+0000 mgr.y (mgr.14391) 208 : cluster [DBG] pgmap v211: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:58.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:58 smithi100 bash[21828]: cluster 2024-07-28T19:31:57.653868+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-07-28T19:31:58.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:58 smithi100 bash[31715]: cluster 2024-07-28T19:31:56.930853+0000 mgr.y (mgr.14391) 208 : cluster [DBG] pgmap v211: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:58.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:58 smithi100 bash[31715]: cluster 2024-07-28T19:31:57.653868+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-07-28T19:31:59.054 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:31:59.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:58 smithi138 bash[23952]: cluster 2024-07-28T19:31:56.930853+0000 mgr.y (mgr.14391) 208 : cluster [DBG] pgmap v211: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:31:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:58 smithi138 bash[23952]: cluster 2024-07-28T19:31:57.653868+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-07-28T19:31:59.223 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:31:59.408 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.Empty (2773 ms) 2024-07-28T19:31:59.408 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.OtherName 2024-07-28T19:31:59.477 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:31:59.504 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:31:59.810 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:31:59.975 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:31:59.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:31:59 smithi100 bash[31715]: cluster 2024-07-28T19:31:58.657918+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-07-28T19:31:59.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:31:59 smithi100 bash[21828]: cluster 2024-07-28T19:31:58.657918+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-07-28T19:32:00.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:31:59 smithi138 bash[23952]: cluster 2024-07-28T19:31:58.657918+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-07-28T19:32:00.156 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.OtherName (749 ms) 2024-07-28T19:32:00.157 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.OtherPool 2024-07-28T19:32:00.230 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:00.256 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:00.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:00 smithi100 bash[21828]: cluster 2024-07-28T19:31:58.931436+0000 mgr.y (mgr.14391) 209 : cluster [DBG] pgmap v214: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:32:00.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:00 smithi100 bash[31715]: cluster 2024-07-28T19:31:58.931436+0000 mgr.y (mgr.14391) 209 : cluster [DBG] pgmap v214: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:32:01.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:00 smithi138 bash[23952]: cluster 2024-07-28T19:31:58.931436+0000 mgr.y (mgr.14391) 209 : cluster [DBG] pgmap v214: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:32:01.709 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:01 smithi100 bash[21828]: cluster 2024-07-28T19:32:00.676501+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-07-28T19:32:01.709 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:01 smithi100 bash[21828]: audit 2024-07-28T19:32:00.911921+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:01.709 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:01 smithi100 bash[31715]: cluster 2024-07-28T19:32:00.676501+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-07-28T19:32:01.709 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:01 smithi100 bash[31715]: audit 2024-07-28T19:32:00.911921+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:01.931 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:02.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:01 smithi138 bash[23952]: cluster 2024-07-28T19:32:00.676501+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-07-28T19:32:02.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:01 smithi138 bash[23952]: audit 2024-07-28T19:32:00.911921+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:02.103 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:02.300 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.OtherPool (2143 ms) 2024-07-28T19:32:02.300 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.OtherNamespace 2024-07-28T19:32:02.359 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:02.382 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:02.692 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:02.891 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:02.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:02 smithi100 bash[21828]: cluster 2024-07-28T19:32:00.938178+0000 mgr.y (mgr.14391) 210 : cluster [DBG] pgmap v216: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:32:02.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:02 smithi100 bash[21828]: cluster 2024-07-28T19:32:01.686589+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-07-28T19:32:02.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:02 smithi100 bash[31715]: cluster 2024-07-28T19:32:00.938178+0000 mgr.y (mgr.14391) 210 : cluster [DBG] pgmap v216: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:32:02.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:02 smithi100 bash[31715]: cluster 2024-07-28T19:32:01.686589+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-07-28T19:32:03.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:02 smithi138 bash[23952]: cluster 2024-07-28T19:32:00.938178+0000 mgr.y (mgr.14391) 210 : cluster [DBG] pgmap v216: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:32:03.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:02 smithi138 bash[23952]: cluster 2024-07-28T19:32:01.686589+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-07-28T19:32:03.081 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.OtherNamespace (782 ms) 2024-07-28T19:32:03.081 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.DataPool 2024-07-28T19:32:03.156 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:03.183 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:03.499 INFO:teuthology.orchestra.run.smithi100.stdout:wait for minority of mons to upgrade 2024-07-28T19:32:03.502 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:03.625 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:03.764 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.DataPool (683 ms) 2024-07-28T19:32:03.764 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-07-28T19:32:03.825 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:03.852 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:04.196 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:32:04.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:03 smithi100 bash[21828]: cluster 2024-07-28T19:32:02.939777+0000 mgr.y (mgr.14391) 211 : cluster [DBG] pgmap v218: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 294 KiB/s rd, 18 KiB/s wr, 373 op/s 2024-07-28T19:32:04.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:03 smithi100 bash[31715]: cluster 2024-07-28T19:32:02.939777+0000 mgr.y (mgr.14391) 211 : cluster [DBG] pgmap v218: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 294 KiB/s rd, 18 KiB/s wr, 373 op/s 2024-07-28T19:32:04.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:03 smithi138 bash[23952]: cluster 2024-07-28T19:32:02.939777+0000 mgr.y (mgr.14391) 211 : cluster [DBG] pgmap v218: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 294 KiB/s rd, 18 KiB/s wr, 373 op/s 2024-07-28T19:32:04.411 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.AbortAfterPrepare (646 ms) 2024-07-28T19:32:04.411 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-07-28T19:32:04.484 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:04.512 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:04.533 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-07-28T19:32:04.724 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (314 ms) 2024-07-28T19:32:04.725 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-07-28T19:32:04.781 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:04.804 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:05.152 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:05.342 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:32:05.590 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.AbortAfterExecute (866 ms) 2024-07-28T19:32:05.591 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-07-28T19:32:05.670 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:05.700 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:05.967 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:06.103 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:32:06.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:05 smithi100 bash[21828]: cluster 2024-07-28T19:32:04.940612+0000 mgr.y (mgr.14391) 212 : cluster [DBG] pgmap v219: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 15 KiB/s wr, 314 op/s 2024-07-28T19:32:06.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:05 smithi100 bash[31715]: cluster 2024-07-28T19:32:04.940612+0000 mgr.y (mgr.14391) 212 : cluster [DBG] pgmap v219: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 15 KiB/s wr, 314 op/s 2024-07-28T19:32:06.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:05 smithi138 bash[23952]: cluster 2024-07-28T19:32:04.940612+0000 mgr.y (mgr.14391) 212 : cluster [DBG] pgmap v219: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 15 KiB/s wr, 314 op/s 2024-07-28T19:32:06.305 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:32:06 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:32:06.307 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (717 ms) 2024-07-28T19:32:06.308 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-07-28T19:32:06.370 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:06.397 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:06.686 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:06.835 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:32:07.047 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (740 ms) 2024-07-28T19:32:07.047 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-07-28T19:32:07.110 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:07.135 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:32:07.150 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.MirroringSamePool (102 ms) 2024-07-28T19:32:07.150 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.MirroringAbort 2024-07-28T19:32:07.215 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:07.242 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:32:07.254 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.MirroringAbort (105 ms) 2024-07-28T19:32:07.254 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-07-28T19:32:07.319 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:07.339 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:32:07.353 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.MirroringOtherPoolDisabled (98 ms) 2024-07-28T19:32:07.353 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-07-28T19:32:07.399 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:07.416 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:32:07.428 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.MirroringOtherPoolEnabled (76 ms) 2024-07-28T19:32:07.428 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.MirroringPool 2024-07-28T19:32:07.477 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:07.497 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:32:07.509 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.MirroringPool (81 ms) 2024-07-28T19:32:07.509 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.Group 2024-07-28T19:32:07.566 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:07.604 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:07.938 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:08.072 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:08.232 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.Group (722 ms) 2024-07-28T19:32:08.232 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.GroupAbort 2024-07-28T19:32:08.297 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:08.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:08 smithi138 bash[23952]: cluster 2024-07-28T19:32:06.941921+0000 mgr.y (mgr.14391) 213 : cluster [DBG] pgmap v220: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 923 op/s 2024-07-28T19:32:08.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:08 smithi138 bash[23952]: cluster 2024-07-28T19:32:07.845797+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-07-28T19:32:08.341 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:08.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:32:08 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:08] "GET /metrics HTTP/1.1" 200 244830 "" "Prometheus/2.43.0" 2024-07-28T19:32:08.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:08 smithi100 bash[21828]: cluster 2024-07-28T19:32:06.941921+0000 mgr.y (mgr.14391) 213 : cluster [DBG] pgmap v220: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 923 op/s 2024-07-28T19:32:08.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:08 smithi100 bash[21828]: cluster 2024-07-28T19:32:07.845797+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-07-28T19:32:08.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:08 smithi100 bash[31715]: cluster 2024-07-28T19:32:06.941921+0000 mgr.y (mgr.14391) 213 : cluster [DBG] pgmap v220: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 923 op/s 2024-07-28T19:32:08.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:08 smithi100 bash[31715]: cluster 2024-07-28T19:32:07.845797+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-07-28T19:32:08.636 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:32:08.843 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.GroupAbort (612 ms) 2024-07-28T19:32:08.843 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.NoSnaps 2024-07-28T19:32:08.897 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:08.918 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:09.001 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:09.075 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:09.327 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:32:09.356 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:09.530 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:09.711 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.NoSnaps (867 ms) 2024-07-28T19:32:09.711 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-07-28T19:32:09.765 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:09.789 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:09.859 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:09.919 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:09.926 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:09.994 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:10.249 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:10.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:10 smithi138 bash[23952]: cluster 2024-07-28T19:32:08.942335+0000 mgr.y (mgr.14391) 214 : cluster [DBG] pgmap v222: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 924 op/s 2024-07-28T19:32:10.420 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:10.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:10 smithi100 bash[21828]: cluster 2024-07-28T19:32:08.942335+0000 mgr.y (mgr.14391) 214 : cluster [DBG] pgmap v222: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 924 op/s 2024-07-28T19:32:10.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:10 smithi100 bash[31715]: cluster 2024-07-28T19:32:08.942335+0000 mgr.y (mgr.14391) 214 : cluster [DBG] pgmap v222: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 924 op/s 2024-07-28T19:32:10.598 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.NoSnapsOtherPool (887 ms) 2024-07-28T19:32:10.598 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-07-28T19:32:10.645 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:10.661 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:10.784 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:11.054 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:11.219 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:11.383 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.NoSnapsDataPool (785 ms) 2024-07-28T19:32:11.383 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-07-28T19:32:11.438 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:11.461 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:11.529 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:11.594 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:11.954 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 41943040 2024-07-28T19:32:11.973 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:12.200 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:12.384 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (1002 ms) 2024-07-28T19:32:12.384 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-07-28T19:32:12.435 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:12.457 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:12.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:12 smithi100 bash[31715]: cluster 2024-07-28T19:32:10.942772+0000 mgr.y (mgr.14391) 215 : cluster [DBG] pgmap v223: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 634 KiB/s rd, 36 KiB/s wr, 798 op/s 2024-07-28T19:32:12.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:12 smithi100 bash[31715]: audit 2024-07-28T19:32:11.062170+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:12.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:12 smithi100 bash[21828]: cluster 2024-07-28T19:32:10.942772+0000 mgr.y (mgr.14391) 215 : cluster [DBG] pgmap v223: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 634 KiB/s rd, 36 KiB/s wr, 798 op/s 2024-07-28T19:32:12.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:12 smithi100 bash[21828]: audit 2024-07-28T19:32:11.062170+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:12.525 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:12.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:12 smithi138 bash[23952]: cluster 2024-07-28T19:32:10.942772+0000 mgr.y (mgr.14391) 215 : cluster [DBG] pgmap v223: 137 pgs: 137 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 634 KiB/s rd, 36 KiB/s wr, 798 op/s 2024-07-28T19:32:12.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:12 smithi138 bash[23952]: audit 2024-07-28T19:32:11.062170+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:12.587 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 0 2024-07-28T19:32:12.612 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:12.804 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:12.915 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:13.045 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (660 ms) 2024-07-28T19:32:13.045 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-07-28T19:32:13.096 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:13.118 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:13.188 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:13.247 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:13.513 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 0 2024-07-28T19:32:13.537 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:13.661 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:13.765 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (721 ms) 2024-07-28T19:32:13.765 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-07-28T19:32:13.815 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:13.834 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:13.920 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:13.982 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:14.235 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:32:14.251 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:14.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:14 smithi100 bash[21828]: cluster 2024-07-28T19:32:12.943533+0000 mgr.y (mgr.14391) 216 : cluster [DBG] pgmap v224: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 21 MiB/s wr, 1.15k op/s 2024-07-28T19:32:14.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:14 smithi100 bash[31715]: cluster 2024-07-28T19:32:12.943533+0000 mgr.y (mgr.14391) 216 : cluster [DBG] pgmap v224: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 21 MiB/s wr, 1.15k op/s 2024-07-28T19:32:14.509 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:14.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:14 smithi138 bash[23952]: cluster 2024-07-28T19:32:12.943533+0000 mgr.y (mgr.14391) 216 : cluster [DBG] pgmap v224: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 21 MiB/s wr, 1.15k op/s 2024-07-28T19:32:14.691 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (925 ms) 2024-07-28T19:32:14.691 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-07-28T19:32:14.747 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:14.765 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:14.837 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:14.902 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:15.196 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: after_prepare_snap 2024-07-28T19:32:16.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:16 smithi100 bash[21828]: cluster 2024-07-28T19:32:14.943909+0000 mgr.y (mgr.14391) 217 : cluster [DBG] pgmap v225: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 21 MiB/s wr, 1.15k op/s 2024-07-28T19:32:16.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:16 smithi100 bash[21828]: audit 2024-07-28T19:32:15.912082+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:16.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:16 smithi100 bash[31715]: cluster 2024-07-28T19:32:14.943909+0000 mgr.y (mgr.14391) 217 : cluster [DBG] pgmap v225: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 21 MiB/s wr, 1.15k op/s 2024-07-28T19:32:16.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:16 smithi100 bash[31715]: audit 2024-07-28T19:32:15.912082+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:16.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:32:16 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:32:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:16 smithi138 bash[23952]: cluster 2024-07-28T19:32:14.943909+0000 mgr.y (mgr.14391) 217 : cluster [DBG] pgmap v225: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 21 MiB/s wr, 1.15k op/s 2024-07-28T19:32:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:16 smithi138 bash[23952]: audit 2024-07-28T19:32:15.912082+0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:17.106 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 41943040 2024-07-28T19:32:17.142 INFO:tasks.workunit.client.0.smithi100.stdout:write: * 0~1000 2024-07-28T19:32:17.153 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:17.187 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:17 smithi100 bash[31715]: cluster 2024-07-28T19:32:16.070888+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-07-28T19:32:17.188 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:17 smithi100 bash[21828]: cluster 2024-07-28T19:32:16.070888+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-07-28T19:32:17.352 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:17.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:17 smithi138 bash[23952]: cluster 2024-07-28T19:32:16.070888+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-07-28T19:32:17.599 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (2908 ms) 2024-07-28T19:32:17.599 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.Snaps 2024-07-28T19:32:17.654 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:17.673 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:17.740 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:32:18.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:32:18 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:18] "GET /metrics HTTP/1.1" 200 245558 "" "Prometheus/2.43.0" 2024-07-28T19:32:18.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:18 smithi100 bash[21828]: cluster 2024-07-28T19:32:16.944791+0000 mgr.y (mgr.14391) 218 : cluster [DBG] pgmap v227: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s rd, 41 MiB/s wr, 1.17k op/s 2024-07-28T19:32:18.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:18 smithi100 bash[21828]: cluster 2024-07-28T19:32:17.089725+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-07-28T19:32:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:18 smithi100 bash[31715]: cluster 2024-07-28T19:32:16.944791+0000 mgr.y (mgr.14391) 218 : cluster [DBG] pgmap v227: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s rd, 41 MiB/s wr, 1.17k op/s 2024-07-28T19:32:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:18 smithi100 bash[31715]: cluster 2024-07-28T19:32:17.089725+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-07-28T19:32:18.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:18 smithi138 bash[23952]: cluster 2024-07-28T19:32:16.944791+0000 mgr.y (mgr.14391) 218 : cluster [DBG] pgmap v227: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s rd, 41 MiB/s wr, 1.17k op/s 2024-07-28T19:32:18.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:18 smithi138 bash[23952]: cluster 2024-07-28T19:32:17.089725+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-07-28T19:32:19.126 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:32:19.158 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:32:19.158 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:32:19.158 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:32:19.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:19 smithi100 bash[21828]: cluster 2024-07-28T19:32:18.098716+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-07-28T19:32:19.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:19 smithi100 bash[31715]: cluster 2024-07-28T19:32:18.098716+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-07-28T19:32:19.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:19 smithi138 bash[23952]: cluster 2024-07-28T19:32:18.098716+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-07-28T19:32:20.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:20 smithi100 bash[21828]: cluster 2024-07-28T19:32:18.945530+0000 mgr.y (mgr.14391) 219 : cluster [DBG] pgmap v230: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 27 MiB/s wr, 769 op/s 2024-07-28T19:32:20.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:20 smithi100 bash[21828]: cluster 2024-07-28T19:32:19.109358+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-07-28T19:32:20.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:20 smithi100 bash[31715]: cluster 2024-07-28T19:32:18.945530+0000 mgr.y (mgr.14391) 219 : cluster [DBG] pgmap v230: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 27 MiB/s wr, 769 op/s 2024-07-28T19:32:20.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:20 smithi100 bash[31715]: cluster 2024-07-28T19:32:19.109358+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-07-28T19:32:20.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:20 smithi138 bash[23952]: cluster 2024-07-28T19:32:18.945530+0000 mgr.y (mgr.14391) 219 : cluster [DBG] pgmap v230: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 27 MiB/s wr, 769 op/s 2024-07-28T19:32:20.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:20 smithi138 bash[23952]: cluster 2024-07-28T19:32:19.109358+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-07-28T19:32:21.148 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:32:21.162 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:32:21.162 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:32:21.182 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:32:21.236 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:32:21.530 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:21 smithi100 bash[31715]: cluster 2024-07-28T19:32:20.111236+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-07-28T19:32:21.531 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:21 smithi100 bash[21828]: cluster 2024-07-28T19:32:20.111236+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-07-28T19:32:21.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:21 smithi138 bash[23952]: cluster 2024-07-28T19:32:20.111236+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-07-28T19:32:22.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:22 smithi100 bash[21828]: cluster 2024-07-28T19:32:20.946252+0000 mgr.y (mgr.14391) 220 : cluster [DBG] pgmap v233: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:22.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:22 smithi100 bash[21828]: cluster 2024-07-28T19:32:21.130424+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-07-28T19:32:22.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:22 smithi100 bash[31715]: cluster 2024-07-28T19:32:20.946252+0000 mgr.y (mgr.14391) 220 : cluster [DBG] pgmap v233: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:22.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:22 smithi100 bash[31715]: cluster 2024-07-28T19:32:21.130424+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-07-28T19:32:22.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:22 smithi138 bash[23952]: cluster 2024-07-28T19:32:20.946252+0000 mgr.y (mgr.14391) 220 : cluster [DBG] pgmap v233: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:22.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:22 smithi138 bash[23952]: cluster 2024-07-28T19:32:21.130424+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-07-28T19:32:23.173 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:23.224 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:32:23.241 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:23.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:23 smithi138 bash[23952]: cluster 2024-07-28T19:32:22.130589+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-07-28T19:32:23.585 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:23 smithi100 bash[21828]: cluster 2024-07-28T19:32:22.130589+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-07-28T19:32:23.585 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:23 smithi100 bash[31715]: cluster 2024-07-28T19:32:22.130589+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-07-28T19:32:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:24 smithi100 bash[21828]: cluster 2024-07-28T19:32:22.946968+0000 mgr.y (mgr.14391) 221 : cluster [DBG] pgmap v236: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 350 op/s 2024-07-28T19:32:24.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:24 smithi100 bash[21828]: cluster 2024-07-28T19:32:23.148849+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-07-28T19:32:24.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:24 smithi100 bash[31715]: cluster 2024-07-28T19:32:22.946968+0000 mgr.y (mgr.14391) 221 : cluster [DBG] pgmap v236: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 350 op/s 2024-07-28T19:32:24.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:24 smithi100 bash[31715]: cluster 2024-07-28T19:32:23.148849+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-07-28T19:32:24.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:24 smithi138 bash[23952]: cluster 2024-07-28T19:32:22.946968+0000 mgr.y (mgr.14391) 221 : cluster [DBG] pgmap v236: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 350 op/s 2024-07-28T19:32:24.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:24 smithi138 bash[23952]: cluster 2024-07-28T19:32:23.148849+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-07-28T19:32:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[21828]: cluster 2024-07-28T19:32:24.153169+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-07-28T19:32:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[21828]: audit 2024-07-28T19:32:24.572358+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.100:0/57297563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[21828]: audit 2024-07-28T19:32:24.949583+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[21828]: audit 2024-07-28T19:32:24.952902+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:32:25.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[21828]: audit 2024-07-28T19:32:24.966384+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:25.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[31715]: cluster 2024-07-28T19:32:24.153169+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-07-28T19:32:25.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[31715]: audit 2024-07-28T19:32:24.572358+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.100:0/57297563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:25.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[31715]: audit 2024-07-28T19:32:24.949583+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:25.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[31715]: audit 2024-07-28T19:32:24.952902+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:32:25.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:25 smithi100 bash[31715]: audit 2024-07-28T19:32:24.966384+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:25.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:25 smithi138 bash[23952]: cluster 2024-07-28T19:32:24.153169+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-07-28T19:32:25.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:25 smithi138 bash[23952]: audit 2024-07-28T19:32:24.572358+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.100:0/57297563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:25.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:25 smithi138 bash[23952]: audit 2024-07-28T19:32:24.949583+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:25.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:25 smithi138 bash[23952]: audit 2024-07-28T19:32:24.952902+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:32:25.546 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:25 smithi138 bash[23952]: audit 2024-07-28T19:32:24.966384+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:32:26.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[21828]: cluster 2024-07-28T19:32:24.947606+0000 mgr.y (mgr.14391) 222 : cluster [DBG] pgmap v239: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 350 op/s 2024-07-28T19:32:26.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[21828]: cephadm 2024-07-28T19:32:24.950626+0000 mgr.y (mgr.14391) 223 : cephadm [INF] Upgrade: Target is version 18.2.2-1765-g1c7dd747 (unknown) 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[21828]: cephadm 2024-07-28T19:32:24.950780+0000 mgr.y (mgr.14391) 224 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09, digests ['quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09'] 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[21828]: cephadm 2024-07-28T19:32:24.968453+0000 mgr.y (mgr.14391) 225 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[21828]: cluster 2024-07-28T19:32:25.158937+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[21828]: cephadm 2024-07-28T19:32:25.225434+0000 mgr.y (mgr.14391) 226 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 on smithi138 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[31715]: cluster 2024-07-28T19:32:24.947606+0000 mgr.y (mgr.14391) 222 : cluster [DBG] pgmap v239: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 350 op/s 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[31715]: cephadm 2024-07-28T19:32:24.950626+0000 mgr.y (mgr.14391) 223 : cephadm [INF] Upgrade: Target is version 18.2.2-1765-g1c7dd747 (unknown) 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[31715]: cephadm 2024-07-28T19:32:24.950780+0000 mgr.y (mgr.14391) 224 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09, digests ['quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09'] 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[31715]: cephadm 2024-07-28T19:32:24.968453+0000 mgr.y (mgr.14391) 225 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-07-28T19:32:26.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[31715]: cluster 2024-07-28T19:32:25.158937+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-07-28T19:32:26.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:26 smithi100 bash[31715]: cephadm 2024-07-28T19:32:25.225434+0000 mgr.y (mgr.14391) 226 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 on smithi138 2024-07-28T19:32:26.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:32:26 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:32:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:26 smithi138 bash[23952]: cluster 2024-07-28T19:32:24.947606+0000 mgr.y (mgr.14391) 222 : cluster [DBG] pgmap v239: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 16 MiB/s wr, 350 op/s 2024-07-28T19:32:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:26 smithi138 bash[23952]: cephadm 2024-07-28T19:32:24.950626+0000 mgr.y (mgr.14391) 223 : cephadm [INF] Upgrade: Target is version 18.2.2-1765-g1c7dd747 (unknown) 2024-07-28T19:32:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:26 smithi138 bash[23952]: cephadm 2024-07-28T19:32:24.950780+0000 mgr.y (mgr.14391) 224 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09, digests ['quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09'] 2024-07-28T19:32:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:26 smithi138 bash[23952]: cephadm 2024-07-28T19:32:24.968453+0000 mgr.y (mgr.14391) 225 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-07-28T19:32:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:26 smithi138 bash[23952]: cluster 2024-07-28T19:32:25.158937+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-07-28T19:32:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:26 smithi138 bash[23952]: cephadm 2024-07-28T19:32:25.225434+0000 mgr.y (mgr.14391) 226 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 on smithi138 2024-07-28T19:32:27.004 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:27.273 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:27.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:27 smithi100 bash[21828]: cluster 2024-07-28T19:32:26.183194+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-07-28T19:32:27.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:27 smithi100 bash[21828]: audit 2024-07-28T19:32:26.978663+0000 mon.a (mon.0) 903 : audit [DBG] from='client.? 172.21.15.100:0/1855522216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:27.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:27 smithi100 bash[31715]: cluster 2024-07-28T19:32:26.183194+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-07-28T19:32:27.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:27 smithi100 bash[31715]: audit 2024-07-28T19:32:26.978663+0000 mon.a (mon.0) 903 : audit [DBG] from='client.? 172.21.15.100:0/1855522216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:27.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:27 smithi138 bash[23952]: cluster 2024-07-28T19:32:26.183194+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-07-28T19:32:27.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:27 smithi138 bash[23952]: audit 2024-07-28T19:32:26.978663+0000 mon.a (mon.0) 903 : audit [DBG] from='client.? 172.21.15.100:0/1855522216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:28.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:28 smithi100 bash[21828]: cluster 2024-07-28T19:32:26.949158+0000 mgr.y (mgr.14391) 227 : cluster [DBG] pgmap v242: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-07-28T19:32:28.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:32:28 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:28] "GET /metrics HTTP/1.1" 200 245558 "" "Prometheus/2.43.0" 2024-07-28T19:32:28.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:28 smithi100 bash[31715]: cluster 2024-07-28T19:32:26.949158+0000 mgr.y (mgr.14391) 227 : cluster [DBG] pgmap v242: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-07-28T19:32:28.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:28 smithi138 bash[23952]: cluster 2024-07-28T19:32:26.949158+0000 mgr.y (mgr.14391) 227 : cluster [DBG] pgmap v242: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-07-28T19:32:29.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:29 smithi100 bash[21828]: cluster 2024-07-28T19:32:28.215064+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-07-28T19:32:29.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:29 smithi100 bash[31715]: cluster 2024-07-28T19:32:28.215064+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-07-28T19:32:29.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:29 smithi138 bash[23952]: cluster 2024-07-28T19:32:28.215064+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-07-28T19:32:30.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:30 smithi100 bash[21828]: cluster 2024-07-28T19:32:28.949893+0000 mgr.y (mgr.14391) 228 : cluster [DBG] pgmap v244: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-07-28T19:32:30.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:30 smithi100 bash[21828]: cluster 2024-07-28T19:32:29.221497+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-07-28T19:32:30.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:30 smithi100 bash[21828]: audit 2024-07-28T19:32:29.338967+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.100:0/4228085956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:30.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:30 smithi100 bash[31715]: cluster 2024-07-28T19:32:28.949893+0000 mgr.y (mgr.14391) 228 : cluster [DBG] pgmap v244: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-07-28T19:32:30.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:30 smithi100 bash[31715]: cluster 2024-07-28T19:32:29.221497+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-07-28T19:32:30.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:30 smithi100 bash[31715]: audit 2024-07-28T19:32:29.338967+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.100:0/4228085956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:30.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:30 smithi138 bash[23952]: cluster 2024-07-28T19:32:28.949893+0000 mgr.y (mgr.14391) 228 : cluster [DBG] pgmap v244: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-07-28T19:32:30.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:30 smithi138 bash[23952]: cluster 2024-07-28T19:32:29.221497+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-07-28T19:32:30.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:30 smithi138 bash[23952]: audit 2024-07-28T19:32:29.338967+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.100:0/4228085956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:30.722 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.Snaps (13124 ms) 2024-07-28T19:32:30.723 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-07-28T19:32:30.794 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:30.815 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:30.908 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:32:31.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:31 smithi100 bash[21828]: cluster 2024-07-28T19:32:30.231339+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-07-28T19:32:31.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:31 smithi100 bash[21828]: audit 2024-07-28T19:32:30.912242+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:31.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:31 smithi100 bash[31715]: cluster 2024-07-28T19:32:30.231339+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-07-28T19:32:31.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:31 smithi100 bash[31715]: audit 2024-07-28T19:32:30.912242+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:31.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:31 smithi138 bash[23952]: cluster 2024-07-28T19:32:30.231339+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-07-28T19:32:31.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:31 smithi138 bash[23952]: audit 2024-07-28T19:32:30.912242+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:32.284 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:32:32.330 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:32:32.330 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:32:32.330 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:32:32.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:32 smithi100 bash[21828]: cluster 2024-07-28T19:32:30.950557+0000 mgr.y (mgr.14391) 229 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-07-28T19:32:32.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:32 smithi100 bash[21828]: cluster 2024-07-28T19:32:31.248857+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-07-28T19:32:32.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:32 smithi100 bash[21828]: audit 2024-07-28T19:32:31.687289+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.100:0/2578062422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:32.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:32 smithi100 bash[31715]: cluster 2024-07-28T19:32:30.950557+0000 mgr.y (mgr.14391) 229 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-07-28T19:32:32.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:32 smithi100 bash[31715]: cluster 2024-07-28T19:32:31.248857+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-07-28T19:32:32.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:32 smithi100 bash[31715]: audit 2024-07-28T19:32:31.687289+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.100:0/2578062422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:32.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:32 smithi138 bash[23952]: cluster 2024-07-28T19:32:30.950557+0000 mgr.y (mgr.14391) 229 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-07-28T19:32:32.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:32 smithi138 bash[23952]: cluster 2024-07-28T19:32:31.248857+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-07-28T19:32:32.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:32 smithi138 bash[23952]: audit 2024-07-28T19:32:31.687289+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.100:0/2578062422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:33.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:33 smithi138 bash[23952]: cluster 2024-07-28T19:32:32.261176+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-07-28T19:32:33.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:33 smithi100 bash[21828]: cluster 2024-07-28T19:32:32.261176+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-07-28T19:32:33.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:33 smithi100 bash[31715]: cluster 2024-07-28T19:32:32.261176+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-07-28T19:32:34.293 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:32:34.311 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:32:34.312 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:32:34.339 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:32:34.389 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:32:34.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:34 smithi138 bash[23952]: cluster 2024-07-28T19:32:32.951306+0000 mgr.y (mgr.14391) 230 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-07-28T19:32:34.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:34 smithi138 bash[23952]: cluster 2024-07-28T19:32:33.269436+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-07-28T19:32:34.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:34 smithi138 bash[23952]: audit 2024-07-28T19:32:34.035342+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.100:0/143881137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:34.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:34 smithi100 bash[21828]: cluster 2024-07-28T19:32:32.951306+0000 mgr.y (mgr.14391) 230 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-07-28T19:32:34.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:34 smithi100 bash[21828]: cluster 2024-07-28T19:32:33.269436+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-07-28T19:32:34.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:34 smithi100 bash[21828]: audit 2024-07-28T19:32:34.035342+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.100:0/143881137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:34.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:34 smithi100 bash[31715]: cluster 2024-07-28T19:32:32.951306+0000 mgr.y (mgr.14391) 230 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-07-28T19:32:34.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:34 smithi100 bash[31715]: cluster 2024-07-28T19:32:33.269436+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-07-28T19:32:34.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:34 smithi100 bash[31715]: audit 2024-07-28T19:32:34.035342+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.100:0/143881137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:35.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:35 smithi138 bash[23952]: cluster 2024-07-28T19:32:34.269070+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-07-28T19:32:35.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:35 smithi100 bash[21828]: cluster 2024-07-28T19:32:34.269070+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-07-28T19:32:35.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:35 smithi100 bash[31715]: cluster 2024-07-28T19:32:34.269070+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-07-28T19:32:36.316 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:36.370 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:32:36.388 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:36.406 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:32:36.485 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:36.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:32:36 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:32:36.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:36 smithi138 bash[23952]: cluster 2024-07-28T19:32:34.952175+0000 mgr.y (mgr.14391) 231 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-07-28T19:32:36.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:36 smithi138 bash[23952]: cluster 2024-07-28T19:32:35.296092+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-07-28T19:32:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:36 smithi100 bash[21828]: cluster 2024-07-28T19:32:34.952175+0000 mgr.y (mgr.14391) 231 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-07-28T19:32:36.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:36 smithi100 bash[21828]: cluster 2024-07-28T19:32:35.296092+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-07-28T19:32:36.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:36 smithi100 bash[31715]: cluster 2024-07-28T19:32:34.952175+0000 mgr.y (mgr.14391) 231 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 265 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-07-28T19:32:36.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:36 smithi100 bash[31715]: cluster 2024-07-28T19:32:35.296092+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-07-28T19:32:37.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:37 smithi138 bash[23952]: cluster 2024-07-28T19:32:36.299781+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-07-28T19:32:37.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:37 smithi138 bash[23952]: audit 2024-07-28T19:32:36.398222+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.100:0/3894918637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:37.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:37 smithi100 bash[21828]: cluster 2024-07-28T19:32:36.299781+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-07-28T19:32:37.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:37 smithi100 bash[21828]: audit 2024-07-28T19:32:36.398222+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.100:0/3894918637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:37.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:37 smithi100 bash[31715]: cluster 2024-07-28T19:32:36.299781+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-07-28T19:32:37.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:37 smithi100 bash[31715]: audit 2024-07-28T19:32:36.398222+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.100:0/3894918637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:38 smithi100 bash[21828]: cluster 2024-07-28T19:32:36.953786+0000 mgr.y (mgr.14391) 232 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-07-28T19:32:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:38 smithi100 bash[21828]: cluster 2024-07-28T19:32:37.315547+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-07-28T19:32:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:38 smithi100 bash[21828]: cluster 2024-07-28T19:32:37.849484+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-07-28T19:32:38.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:32:38 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:38] "GET /metrics HTTP/1.1" 200 245593 "" "Prometheus/2.43.0" 2024-07-28T19:32:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:38 smithi100 bash[31715]: cluster 2024-07-28T19:32:36.953786+0000 mgr.y (mgr.14391) 232 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-07-28T19:32:38.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:38 smithi100 bash[31715]: cluster 2024-07-28T19:32:37.315547+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-07-28T19:32:38.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:38 smithi100 bash[31715]: cluster 2024-07-28T19:32:37.849484+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-07-28T19:32:38.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:38 smithi138 bash[23952]: cluster 2024-07-28T19:32:36.953786+0000 mgr.y (mgr.14391) 232 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 155 KiB/s rd, 16 MiB/s wr, 228 op/s 2024-07-28T19:32:38.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:38 smithi138 bash[23952]: cluster 2024-07-28T19:32:37.315547+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-07-28T19:32:38.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:38 smithi138 bash[23952]: cluster 2024-07-28T19:32:37.849484+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-07-28T19:32:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:39 smithi100 bash[21828]: audit 2024-07-28T19:32:38.749366+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.100:0/2393799688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:39 smithi100 bash[21828]: cluster 2024-07-28T19:32:38.851540+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-07-28T19:32:39.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:39 smithi100 bash[31715]: audit 2024-07-28T19:32:38.749366+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.100:0/2393799688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:39.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:39 smithi100 bash[31715]: cluster 2024-07-28T19:32:38.851540+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-07-28T19:32:39.761 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:39.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:39 smithi138 bash[23952]: audit 2024-07-28T19:32:38.749366+0000 mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.100:0/2393799688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:39.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:39 smithi138 bash[23952]: cluster 2024-07-28T19:32:38.851540+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-07-28T19:32:40.068 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:40.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:40 smithi100 bash[21828]: cluster 2024-07-28T19:32:38.954606+0000 mgr.y (mgr.14391) 233 : cluster [DBG] pgmap v260: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 KiB/s rd, 18 MiB/s wr, 249 op/s 2024-07-28T19:32:40.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:40 smithi100 bash[31715]: cluster 2024-07-28T19:32:38.954606+0000 mgr.y (mgr.14391) 233 : cluster [DBG] pgmap v260: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 KiB/s rd, 18 MiB/s wr, 249 op/s 2024-07-28T19:32:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:40 smithi138 bash[23952]: cluster 2024-07-28T19:32:38.954606+0000 mgr.y (mgr.14391) 233 : cluster [DBG] pgmap v260: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 170 KiB/s rd, 18 MiB/s wr, 249 op/s 2024-07-28T19:32:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:41 smithi100 bash[21828]: cluster 2024-07-28T19:32:40.856412+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-07-28T19:32:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:41 smithi100 bash[21828]: cluster 2024-07-28T19:32:40.955260+0000 mgr.y (mgr.14391) 234 : cluster [DBG] pgmap v262: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:41 smithi100 bash[21828]: audit 2024-07-28T19:32:41.111722+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.100:0/1544782634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:41 smithi100 bash[31715]: cluster 2024-07-28T19:32:40.856412+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-07-28T19:32:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:41 smithi100 bash[31715]: cluster 2024-07-28T19:32:40.955260+0000 mgr.y (mgr.14391) 234 : cluster [DBG] pgmap v262: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:41 smithi100 bash[31715]: audit 2024-07-28T19:32:41.111722+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.100:0/1544782634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:41 smithi138 bash[23952]: cluster 2024-07-28T19:32:40.856412+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-07-28T19:32:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:41 smithi138 bash[23952]: cluster 2024-07-28T19:32:40.955260+0000 mgr.y (mgr.14391) 234 : cluster [DBG] pgmap v262: 137 pgs: 137 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:41 smithi138 bash[23952]: audit 2024-07-28T19:32:41.111722+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.100:0/1544782634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:43.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:42 smithi100 bash[31715]: cluster 2024-07-28T19:32:41.860622+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-07-28T19:32:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:42 smithi100 bash[21828]: cluster 2024-07-28T19:32:41.860622+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-07-28T19:32:43.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:42 smithi138 bash[23952]: cluster 2024-07-28T19:32:41.860622+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-07-28T19:32:43.390 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsOtherPool (12667 ms) 2024-07-28T19:32:43.390 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-07-28T19:32:43.454 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:43.477 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:43.558 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:32:44.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:43 smithi100 bash[21828]: cluster 2024-07-28T19:32:42.874867+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-07-28T19:32:44.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:43 smithi100 bash[21828]: cluster 2024-07-28T19:32:42.956007+0000 mgr.y (mgr.14391) 235 : cluster [DBG] pgmap v265: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s rd, 9.8 MiB/s wr, 368 op/s 2024-07-28T19:32:44.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:43 smithi100 bash[21828]: audit 2024-07-28T19:32:43.500680+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.100:0/2801411568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:44.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:43 smithi100 bash[31715]: cluster 2024-07-28T19:32:42.874867+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-07-28T19:32:44.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:43 smithi100 bash[31715]: cluster 2024-07-28T19:32:42.956007+0000 mgr.y (mgr.14391) 235 : cluster [DBG] pgmap v265: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s rd, 9.8 MiB/s wr, 368 op/s 2024-07-28T19:32:44.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:43 smithi100 bash[31715]: audit 2024-07-28T19:32:43.500680+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.100:0/2801411568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:44.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:43 smithi138 bash[23952]: cluster 2024-07-28T19:32:42.874867+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-07-28T19:32:44.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:43 smithi138 bash[23952]: cluster 2024-07-28T19:32:42.956007+0000 mgr.y (mgr.14391) 235 : cluster [DBG] pgmap v265: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s rd, 9.8 MiB/s wr, 368 op/s 2024-07-28T19:32:44.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:43 smithi138 bash[23952]: audit 2024-07-28T19:32:43.500680+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.100:0/2801411568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:44.920 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:32:44.963 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:32:44.963 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:32:44.964 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:32:45.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:44 smithi100 bash[21828]: cluster 2024-07-28T19:32:43.894734+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-07-28T19:32:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:44 smithi100 bash[31715]: cluster 2024-07-28T19:32:43.894734+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-07-28T19:32:45.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:44 smithi138 bash[23952]: cluster 2024-07-28T19:32:43.894734+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-07-28T19:32:46.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:45 smithi100 bash[21828]: cluster 2024-07-28T19:32:44.906210+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-07-28T19:32:46.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:45 smithi100 bash[21828]: cluster 2024-07-28T19:32:44.956961+0000 mgr.y (mgr.14391) 236 : cluster [DBG] pgmap v268: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-07-28T19:32:46.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:45 smithi100 bash[21828]: audit 2024-07-28T19:32:45.859412+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.100:0/2295287007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:46.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:45 smithi100 bash[31715]: cluster 2024-07-28T19:32:44.906210+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-07-28T19:32:46.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:45 smithi100 bash[31715]: cluster 2024-07-28T19:32:44.956961+0000 mgr.y (mgr.14391) 236 : cluster [DBG] pgmap v268: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-07-28T19:32:46.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:45 smithi100 bash[31715]: audit 2024-07-28T19:32:45.859412+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.100:0/2295287007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:46.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:32:46 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:32:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:45 smithi138 bash[23952]: cluster 2024-07-28T19:32:44.906210+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-07-28T19:32:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:45 smithi138 bash[23952]: cluster 2024-07-28T19:32:44.956961+0000 mgr.y (mgr.14391) 236 : cluster [DBG] pgmap v268: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-07-28T19:32:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:45 smithi138 bash[23952]: audit 2024-07-28T19:32:45.859412+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.100:0/2295287007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:46.952 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:32:46.964 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:32:46.965 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:32:46.982 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:32:47.025 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:32:47.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:46 smithi100 bash[21828]: audit 2024-07-28T19:32:45.914841+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:47.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:46 smithi100 bash[21828]: cluster 2024-07-28T19:32:45.920169+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-07-28T19:32:47.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:46 smithi100 bash[31715]: audit 2024-07-28T19:32:45.914841+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:47.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:46 smithi100 bash[31715]: cluster 2024-07-28T19:32:45.920169+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-07-28T19:32:47.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:46 smithi138 bash[23952]: audit 2024-07-28T19:32:45.914841+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:32:47.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:46 smithi138 bash[23952]: cluster 2024-07-28T19:32:45.920169+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-07-28T19:32:48.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:32:48 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:48] "GET /metrics HTTP/1.1" 200 245707 "" "Prometheus/2.43.0" 2024-07-28T19:32:48.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:47 smithi100 bash[31715]: cluster 2024-07-28T19:32:46.931395+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-07-28T19:32:48.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:47 smithi100 bash[31715]: cluster 2024-07-28T19:32:46.957773+0000 mgr.y (mgr.14391) 237 : cluster [DBG] pgmap v271: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 129 op/s 2024-07-28T19:32:48.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:47 smithi100 bash[31715]: cluster 2024-07-28T19:32:47.852979+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-07-28T19:32:48.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:47 smithi100 bash[21828]: cluster 2024-07-28T19:32:46.931395+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-07-28T19:32:48.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:47 smithi100 bash[21828]: cluster 2024-07-28T19:32:46.957773+0000 mgr.y (mgr.14391) 237 : cluster [DBG] pgmap v271: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 129 op/s 2024-07-28T19:32:48.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:47 smithi100 bash[21828]: cluster 2024-07-28T19:32:47.852979+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-07-28T19:32:48.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:47 smithi138 bash[23952]: cluster 2024-07-28T19:32:46.931395+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-07-28T19:32:48.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:47 smithi138 bash[23952]: cluster 2024-07-28T19:32:46.957773+0000 mgr.y (mgr.14391) 237 : cluster [DBG] pgmap v271: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 129 op/s 2024-07-28T19:32:48.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:47 smithi138 bash[23952]: cluster 2024-07-28T19:32:47.852979+0000 mon.a (mon.0) 929 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-07-28T19:32:48.869 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:48.916 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:32:48.937 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:32:49.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:48 smithi100 bash[21828]: audit 2024-07-28T19:32:48.210951+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.100:0/4128487522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:49.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:48 smithi100 bash[21828]: cluster 2024-07-28T19:32:48.856038+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-07-28T19:32:49.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:48 smithi100 bash[31715]: audit 2024-07-28T19:32:48.210951+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.100:0/4128487522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:49.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:48 smithi100 bash[31715]: cluster 2024-07-28T19:32:48.856038+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-07-28T19:32:49.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:48 smithi138 bash[23952]: audit 2024-07-28T19:32:48.210951+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.100:0/4128487522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:49.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:48 smithi138 bash[23952]: cluster 2024-07-28T19:32:48.856038+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-07-28T19:32:50.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:49 smithi100 bash[21828]: cluster 2024-07-28T19:32:48.958531+0000 mgr.y (mgr.14391) 238 : cluster [DBG] pgmap v274: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 129 op/s 2024-07-28T19:32:50.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:49 smithi100 bash[31715]: cluster 2024-07-28T19:32:48.958531+0000 mgr.y (mgr.14391) 238 : cluster [DBG] pgmap v274: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 129 op/s 2024-07-28T19:32:50.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:49 smithi138 bash[23952]: cluster 2024-07-28T19:32:48.958531+0000 mgr.y (mgr.14391) 238 : cluster [DBG] pgmap v274: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 8.0 MiB/s wr, 129 op/s 2024-07-28T19:32:51.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:50 smithi100 bash[21828]: cluster 2024-07-28T19:32:49.954032+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-07-28T19:32:51.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:50 smithi100 bash[21828]: audit 2024-07-28T19:32:50.558298+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.100:0/2748912328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:50 smithi100 bash[31715]: cluster 2024-07-28T19:32:49.954032+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-07-28T19:32:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:50 smithi100 bash[31715]: audit 2024-07-28T19:32:50.558298+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.100:0/2748912328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:50 smithi138 bash[23952]: cluster 2024-07-28T19:32:49.954032+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-07-28T19:32:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:50 smithi138 bash[23952]: audit 2024-07-28T19:32:50.558298+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.100:0/2748912328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:52.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:51 smithi100 bash[21828]: cluster 2024-07-28T19:32:50.954736+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-07-28T19:32:52.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:51 smithi100 bash[21828]: cluster 2024-07-28T19:32:50.959482+0000 mgr.y (mgr.14391) 239 : cluster [DBG] pgmap v277: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:52.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:51 smithi100 bash[31715]: cluster 2024-07-28T19:32:50.954736+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-07-28T19:32:52.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:51 smithi100 bash[31715]: cluster 2024-07-28T19:32:50.959482+0000 mgr.y (mgr.14391) 239 : cluster [DBG] pgmap v277: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:51 smithi138 bash[23952]: cluster 2024-07-28T19:32:50.954736+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-07-28T19:32:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:51 smithi138 bash[23952]: cluster 2024-07-28T19:32:50.959482+0000 mgr.y (mgr.14391) 239 : cluster [DBG] pgmap v277: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:32:52.737 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:32:52.932 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:32:53.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:52 smithi100 bash[21828]: cluster 2024-07-28T19:32:51.979419+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-07-28T19:32:53.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:52 smithi100 bash[21828]: cluster 2024-07-28T19:32:52.859911+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-07-28T19:32:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:52 smithi100 bash[31715]: cluster 2024-07-28T19:32:51.979419+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-07-28T19:32:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:52 smithi100 bash[31715]: cluster 2024-07-28T19:32:52.859911+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-07-28T19:32:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:52 smithi138 bash[23952]: cluster 2024-07-28T19:32:51.979419+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-07-28T19:32:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:52 smithi138 bash[23952]: cluster 2024-07-28T19:32:52.859911+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-07-28T19:32:54.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:53 smithi100 bash[21828]: cluster 2024-07-28T19:32:52.959954+0000 mgr.y (mgr.14391) 240 : cluster [DBG] pgmap v280: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-07-28T19:32:54.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:53 smithi100 bash[21828]: audit 2024-07-28T19:32:52.976190+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.100:0/1726862002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:54.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:53 smithi100 bash[31715]: cluster 2024-07-28T19:32:52.959954+0000 mgr.y (mgr.14391) 240 : cluster [DBG] pgmap v280: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-07-28T19:32:54.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:53 smithi100 bash[31715]: audit 2024-07-28T19:32:52.976190+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.100:0/1726862002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:54.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:53 smithi138 bash[23952]: cluster 2024-07-28T19:32:52.959954+0000 mgr.y (mgr.14391) 240 : cluster [DBG] pgmap v280: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-07-28T19:32:54.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:53 smithi138 bash[23952]: audit 2024-07-28T19:32:52.976190+0000 mon.a (mon.0) 935 : audit [DBG] from='client.? 172.21.15.100:0/1726862002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:55.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:54 smithi100 bash[21828]: cluster 2024-07-28T19:32:53.984975+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-07-28T19:32:55.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:54 smithi100 bash[31715]: cluster 2024-07-28T19:32:53.984975+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-07-28T19:32:55.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:54 smithi138 bash[23952]: cluster 2024-07-28T19:32:53.984975+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-07-28T19:32:56.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:55 smithi100 bash[21828]: cluster 2024-07-28T19:32:54.960689+0000 mgr.y (mgr.14391) 241 : cluster [DBG] pgmap v282: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-07-28T19:32:56.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:55 smithi100 bash[21828]: cluster 2024-07-28T19:32:54.996681+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-07-28T19:32:56.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:55 smithi100 bash[21828]: audit 2024-07-28T19:32:55.330311+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.100:0/2281640626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:56.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:55 smithi100 bash[31715]: cluster 2024-07-28T19:32:54.960689+0000 mgr.y (mgr.14391) 241 : cluster [DBG] pgmap v282: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-07-28T19:32:56.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:55 smithi100 bash[31715]: cluster 2024-07-28T19:32:54.996681+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-07-28T19:32:56.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:55 smithi100 bash[31715]: audit 2024-07-28T19:32:55.330311+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.100:0/2281640626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:56.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:32:56 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:32:56.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:55 smithi138 bash[23952]: cluster 2024-07-28T19:32:54.960689+0000 mgr.y (mgr.14391) 241 : cluster [DBG] pgmap v282: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-07-28T19:32:56.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:55 smithi138 bash[23952]: cluster 2024-07-28T19:32:54.996681+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-07-28T19:32:56.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:55 smithi138 bash[23952]: audit 2024-07-28T19:32:55.330311+0000 mon.a (mon.0) 938 : audit [DBG] from='client.? 172.21.15.100:0/2281640626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:56.527 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsDataPool (13137 ms) 2024-07-28T19:32:56.527 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-07-28T19:32:56.593 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:32:56.615 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:32:56.721 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:32:57.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:57 smithi138 bash[23952]: cluster 2024-07-28T19:32:56.008602+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-07-28T19:32:57.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:57 smithi100 bash[21828]: cluster 2024-07-28T19:32:56.008602+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-07-28T19:32:57.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:57 smithi100 bash[31715]: cluster 2024-07-28T19:32:56.008602+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-07-28T19:32:57.878 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:32:57.927 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:32:57.927 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:32:57.927 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:32:58.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:58 smithi138 bash[23952]: cluster 2024-07-28T19:32:56.962292+0000 mgr.y (mgr.14391) 242 : cluster [DBG] pgmap v285: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s rd, 9.8 MiB/s wr, 476 op/s 2024-07-28T19:32:58.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:58 smithi138 bash[23952]: cluster 2024-07-28T19:32:57.016564+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-07-28T19:32:58.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:58 smithi138 bash[23952]: audit 2024-07-28T19:32:57.679907+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.100:0/2450955995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:58.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:58 smithi138 bash[23952]: cluster 2024-07-28T19:32:57.862853+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-07-28T19:32:58.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:32:58] "GET /metrics HTTP/1.1" 200 245707 "" "Prometheus/2.43.0" 2024-07-28T19:32:58.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[31715]: cluster 2024-07-28T19:32:56.962292+0000 mgr.y (mgr.14391) 242 : cluster [DBG] pgmap v285: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s rd, 9.8 MiB/s wr, 476 op/s 2024-07-28T19:32:58.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[31715]: cluster 2024-07-28T19:32:57.016564+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-07-28T19:32:58.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[31715]: audit 2024-07-28T19:32:57.679907+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.100:0/2450955995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:58.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[31715]: cluster 2024-07-28T19:32:57.862853+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-07-28T19:32:58.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[21828]: cluster 2024-07-28T19:32:56.962292+0000 mgr.y (mgr.14391) 242 : cluster [DBG] pgmap v285: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s rd, 9.8 MiB/s wr, 476 op/s 2024-07-28T19:32:58.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[21828]: cluster 2024-07-28T19:32:57.016564+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-07-28T19:32:58.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[21828]: audit 2024-07-28T19:32:57.679907+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.100:0/2450955995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:32:58.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:58 smithi100 bash[21828]: cluster 2024-07-28T19:32:57.862853+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-07-28T19:32:59.884 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:32:59.901 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:32:59.901 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:32:59.926 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:32:59.971 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:33:00.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:59 smithi100 bash[21828]: cluster 2024-07-28T19:32:58.860389+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-07-28T19:33:00.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:32:59 smithi100 bash[21828]: cluster 2024-07-28T19:32:58.963041+0000 mgr.y (mgr.14391) 243 : cluster [DBG] pgmap v289: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 71 MiB/s rd, 10 MiB/s wr, 491 op/s 2024-07-28T19:33:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:59 smithi100 bash[31715]: cluster 2024-07-28T19:32:58.860389+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-07-28T19:33:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:32:59 smithi100 bash[31715]: cluster 2024-07-28T19:32:58.963041+0000 mgr.y (mgr.14391) 243 : cluster [DBG] pgmap v289: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 71 MiB/s rd, 10 MiB/s wr, 491 op/s 2024-07-28T19:33:00.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:59 smithi138 bash[23952]: cluster 2024-07-28T19:32:58.860389+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-07-28T19:33:00.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:32:59 smithi138 bash[23952]: cluster 2024-07-28T19:32:58.963041+0000 mgr.y (mgr.14391) 243 : cluster [DBG] pgmap v289: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 71 MiB/s rd, 10 MiB/s wr, 491 op/s 2024-07-28T19:33:01.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:00 smithi100 bash[21828]: cluster 2024-07-28T19:32:59.869488+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-07-28T19:33:01.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:00 smithi100 bash[21828]: audit 2024-07-28T19:33:00.055742+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.100:0/4199048171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:00 smithi100 bash[31715]: cluster 2024-07-28T19:32:59.869488+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-07-28T19:33:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:00 smithi100 bash[31715]: audit 2024-07-28T19:33:00.055742+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.100:0/4199048171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:01.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:00 smithi138 bash[23952]: cluster 2024-07-28T19:32:59.869488+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-07-28T19:33:01.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:00 smithi138 bash[23952]: audit 2024-07-28T19:33:00.055742+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.100:0/4199048171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:01.908 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:01.963 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:33:01.983 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:33:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:01 smithi100 bash[21828]: cluster 2024-07-28T19:33:00.877127+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-07-28T19:33:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:01 smithi100 bash[21828]: audit 2024-07-28T19:33:00.913245+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:01 smithi100 bash[21828]: cluster 2024-07-28T19:33:00.963721+0000 mgr.y (mgr.14391) 244 : cluster [DBG] pgmap v292: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:02.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:01 smithi100 bash[31715]: cluster 2024-07-28T19:33:00.877127+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-07-28T19:33:02.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:01 smithi100 bash[31715]: audit 2024-07-28T19:33:00.913245+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:01 smithi100 bash[31715]: cluster 2024-07-28T19:33:00.963721+0000 mgr.y (mgr.14391) 244 : cluster [DBG] pgmap v292: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:01 smithi138 bash[23952]: cluster 2024-07-28T19:33:00.877127+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-07-28T19:33:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:01 smithi138 bash[23952]: audit 2024-07-28T19:33:00.913245+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:01 smithi138 bash[23952]: cluster 2024-07-28T19:33:00.963721+0000 mgr.y (mgr.14391) 244 : cluster [DBG] pgmap v292: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:02 smithi100 bash[21828]: cluster 2024-07-28T19:33:01.892096+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-07-28T19:33:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:02 smithi100 bash[21828]: audit 2024-07-28T19:33:02.402068+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.100:0/671091626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:03.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:02 smithi100 bash[31715]: cluster 2024-07-28T19:33:01.892096+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-07-28T19:33:03.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:02 smithi100 bash[31715]: audit 2024-07-28T19:33:02.402068+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.100:0/671091626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:03.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:02 smithi138 bash[23952]: cluster 2024-07-28T19:33:01.892096+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-07-28T19:33:03.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:02 smithi138 bash[23952]: audit 2024-07-28T19:33:02.402068+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.100:0/671091626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:04.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:03 smithi100 bash[21828]: cluster 2024-07-28T19:33:02.899522+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-07-28T19:33:04.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:03 smithi100 bash[21828]: cluster 2024-07-28T19:33:02.964535+0000 mgr.y (mgr.14391) 245 : cluster [DBG] pgmap v295: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:04.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:03 smithi100 bash[31715]: cluster 2024-07-28T19:33:02.899522+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-07-28T19:33:04.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:03 smithi100 bash[31715]: cluster 2024-07-28T19:33:02.964535+0000 mgr.y (mgr.14391) 245 : cluster [DBG] pgmap v295: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:04.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:03 smithi138 bash[23952]: cluster 2024-07-28T19:33:02.899522+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-07-28T19:33:04.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:03 smithi138 bash[23952]: cluster 2024-07-28T19:33:02.964535+0000 mgr.y (mgr.14391) 245 : cluster [DBG] pgmap v295: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:05.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:04 smithi100 bash[21828]: cluster 2024-07-28T19:33:03.909483+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-07-28T19:33:05.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:04 smithi100 bash[21828]: audit 2024-07-28T19:33:04.750840+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.100:0/3681632547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:05.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:04 smithi100 bash[31715]: cluster 2024-07-28T19:33:03.909483+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-07-28T19:33:05.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:04 smithi100 bash[31715]: audit 2024-07-28T19:33:04.750840+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.100:0/3681632547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:05.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:04 smithi138 bash[23952]: cluster 2024-07-28T19:33:03.909483+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-07-28T19:33:05.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:04 smithi138 bash[23952]: audit 2024-07-28T19:33:04.750840+0000 mon.a (mon.0) 951 : audit [DBG] from='client.? 172.21.15.100:0/3681632547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:05.829 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 41943040 2024-07-28T19:33:05.888 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:33:06.171 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:33:06.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:05 smithi100 bash[21828]: cluster 2024-07-28T19:33:04.918050+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-07-28T19:33:06.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:05 smithi100 bash[21828]: cluster 2024-07-28T19:33:04.965383+0000 mgr.y (mgr.14391) 246 : cluster [DBG] pgmap v298: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:06.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:05 smithi100 bash[31715]: cluster 2024-07-28T19:33:04.918050+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-07-28T19:33:06.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:05 smithi100 bash[31715]: cluster 2024-07-28T19:33:04.965383+0000 mgr.y (mgr.14391) 246 : cluster [DBG] pgmap v298: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:06.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:06 smithi138 bash[24937]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:33:06.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:05 smithi138 bash[23952]: cluster 2024-07-28T19:33:04.918050+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-07-28T19:33:06.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:05 smithi138 bash[23952]: cluster 2024-07-28T19:33:04.965383+0000 mgr.y (mgr.14391) 246 : cluster [DBG] pgmap v298: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:08.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:08 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:08] "GET /metrics HTTP/1.1" 200 245779 "" "Prometheus/2.43.0" 2024-07-28T19:33:08.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:07 smithi100 bash[21828]: cluster 2024-07-28T19:33:06.930922+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-07-28T19:33:08.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:07 smithi100 bash[21828]: cluster 2024-07-28T19:33:06.965984+0000 mgr.y (mgr.14391) 247 : cluster [DBG] pgmap v300: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 5.9 MiB/s wr, 498 op/s 2024-07-28T19:33:08.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:07 smithi100 bash[21828]: audit 2024-07-28T19:33:07.111946+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.100:0/931904281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:07 smithi100 bash[31715]: cluster 2024-07-28T19:33:06.930922+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-07-28T19:33:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:07 smithi100 bash[31715]: cluster 2024-07-28T19:33:06.965984+0000 mgr.y (mgr.14391) 247 : cluster [DBG] pgmap v300: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 5.9 MiB/s wr, 498 op/s 2024-07-28T19:33:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:07 smithi100 bash[31715]: audit 2024-07-28T19:33:07.111946+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.100:0/931904281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:08.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:07 smithi138 bash[23952]: cluster 2024-07-28T19:33:06.930922+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-07-28T19:33:08.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:07 smithi138 bash[23952]: cluster 2024-07-28T19:33:06.965984+0000 mgr.y (mgr.14391) 247 : cluster [DBG] pgmap v300: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 5.9 MiB/s wr, 498 op/s 2024-07-28T19:33:08.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:07 smithi138 bash[23952]: audit 2024-07-28T19:33:07.111946+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.100:0/931904281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:09.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:08 smithi100 bash[21828]: cluster 2024-07-28T19:33:07.941872+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-07-28T19:33:09.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:08 smithi100 bash[31715]: cluster 2024-07-28T19:33:07.941872+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-07-28T19:33:09.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:08 smithi138 bash[23952]: cluster 2024-07-28T19:33:07.941872+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-07-28T19:33:09.456 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (12929 ms) 2024-07-28T19:33:09.456 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-07-28T19:33:09.519 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:09.538 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:33:09.638 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:33:10.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:09 smithi100 bash[21828]: cluster 2024-07-28T19:33:08.953231+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-07-28T19:33:10.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:09 smithi100 bash[21828]: cluster 2024-07-28T19:33:08.966821+0000 mgr.y (mgr.14391) 248 : cluster [DBG] pgmap v303: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 501 op/s 2024-07-28T19:33:10.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:09 smithi100 bash[21828]: audit 2024-07-28T19:33:09.488170+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.100:0/1761406076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:10.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:09 smithi100 bash[31715]: cluster 2024-07-28T19:33:08.953231+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-07-28T19:33:10.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:09 smithi100 bash[31715]: cluster 2024-07-28T19:33:08.966821+0000 mgr.y (mgr.14391) 248 : cluster [DBG] pgmap v303: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 501 op/s 2024-07-28T19:33:10.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:09 smithi100 bash[31715]: audit 2024-07-28T19:33:09.488170+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.100:0/1761406076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:10.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:09 smithi138 bash[23952]: cluster 2024-07-28T19:33:08.953231+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-07-28T19:33:10.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:09 smithi138 bash[23952]: cluster 2024-07-28T19:33:08.966821+0000 mgr.y (mgr.14391) 248 : cluster [DBG] pgmap v303: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 501 op/s 2024-07-28T19:33:10.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:09 smithi138 bash[23952]: audit 2024-07-28T19:33:09.488170+0000 mon.c (mon.2) 153 : audit [DBG] from='client.? 172.21.15.100:0/1761406076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:10.991 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:33:11.021 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:33:11.021 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:33:11.021 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:33:11.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:10 smithi100 bash[21828]: cluster 2024-07-28T19:33:09.963434+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-07-28T19:33:11.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:10 smithi100 bash[31715]: cluster 2024-07-28T19:33:09.963434+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-07-28T19:33:11.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:10 smithi138 bash[23952]: cluster 2024-07-28T19:33:09.963434+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-07-28T19:33:12.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:11 smithi138 bash[23952]: cluster 2024-07-28T19:33:10.963890+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-07-28T19:33:12.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:11 smithi138 bash[23952]: cluster 2024-07-28T19:33:10.967734+0000 mgr.y (mgr.14391) 249 : cluster [DBG] pgmap v306: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:12.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:11 smithi138 bash[23952]: audit 2024-07-28T19:33:11.833129+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.100:0/2427023214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:12.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:11 smithi100 bash[21828]: cluster 2024-07-28T19:33:10.963890+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-07-28T19:33:12.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:11 smithi100 bash[21828]: cluster 2024-07-28T19:33:10.967734+0000 mgr.y (mgr.14391) 249 : cluster [DBG] pgmap v306: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:12.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:11 smithi100 bash[21828]: audit 2024-07-28T19:33:11.833129+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.100:0/2427023214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:11 smithi100 bash[31715]: cluster 2024-07-28T19:33:10.963890+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-07-28T19:33:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:11 smithi100 bash[31715]: cluster 2024-07-28T19:33:10.967734+0000 mgr.y (mgr.14391) 249 : cluster [DBG] pgmap v306: 137 pgs: 137 active+clean; 477 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:12.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:11 smithi100 bash[31715]: audit 2024-07-28T19:33:11.833129+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.100:0/2427023214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:13.009 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:33:13.024 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:33:13.024 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:33:13.041 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:33:13.095 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:33:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[21828]: cluster 2024-07-28T19:33:11.986525+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-07-28T19:33:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[21828]: audit 2024-07-28T19:33:12.737856+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[21828]: audit 2024-07-28T19:33:12.740503+0000 mon.c (mon.2) 154 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:33:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[21828]: audit 2024-07-28T19:33:12.741344+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:33:13.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[21828]: audit 2024-07-28T19:33:12.743706+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:33:13.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[21828]: audit 2024-07-28T19:33:12.745665+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[31715]: cluster 2024-07-28T19:33:11.986525+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-07-28T19:33:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[31715]: audit 2024-07-28T19:33:12.737856+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[31715]: audit 2024-07-28T19:33:12.740503+0000 mon.c (mon.2) 154 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:33:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[31715]: audit 2024-07-28T19:33:12.741344+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:33:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[31715]: audit 2024-07-28T19:33:12.743706+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:33:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:12 smithi100 bash[31715]: audit 2024-07-28T19:33:12.745665+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:13.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:12 smithi138 bash[23952]: cluster 2024-07-28T19:33:11.986525+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-07-28T19:33:13.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:12 smithi138 bash[23952]: audit 2024-07-28T19:33:12.737856+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:13.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:12 smithi138 bash[23952]: audit 2024-07-28T19:33:12.740503+0000 mon.c (mon.2) 154 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:33:13.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:12 smithi138 bash[23952]: audit 2024-07-28T19:33:12.741344+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:33:13.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:12 smithi138 bash[23952]: audit 2024-07-28T19:33:12.743706+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:33:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:12 smithi138 bash[23952]: audit 2024-07-28T19:33:12.745665+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:14.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:14 smithi138 bash[23952]: cephadm 2024-07-28T19:33:12.727898+0000 mgr.y (mgr.14391) 250 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-28T19:33:14.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:14 smithi138 bash[23952]: cephadm 2024-07-28T19:33:12.747655+0000 mgr.y (mgr.14391) 251 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:33:14.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:14 smithi138 bash[23952]: cluster 2024-07-28T19:33:12.968525+0000 mgr.y (mgr.14391) 252 : cluster [DBG] pgmap v308: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 154 op/s 2024-07-28T19:33:14.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:14 smithi138 bash[23952]: cluster 2024-07-28T19:33:12.992567+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-07-28T19:33:14.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:14 smithi100 bash[21828]: cephadm 2024-07-28T19:33:12.727898+0000 mgr.y (mgr.14391) 250 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-28T19:33:14.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:14 smithi100 bash[21828]: cephadm 2024-07-28T19:33:12.747655+0000 mgr.y (mgr.14391) 251 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:33:14.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:14 smithi100 bash[21828]: cluster 2024-07-28T19:33:12.968525+0000 mgr.y (mgr.14391) 252 : cluster [DBG] pgmap v308: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 154 op/s 2024-07-28T19:33:14.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:14 smithi100 bash[21828]: cluster 2024-07-28T19:33:12.992567+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-07-28T19:33:14.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:14 smithi100 bash[31715]: cephadm 2024-07-28T19:33:12.727898+0000 mgr.y (mgr.14391) 250 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-28T19:33:14.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:14 smithi100 bash[31715]: cephadm 2024-07-28T19:33:12.747655+0000 mgr.y (mgr.14391) 251 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:33:14.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:14 smithi100 bash[31715]: cluster 2024-07-28T19:33:12.968525+0000 mgr.y (mgr.14391) 252 : cluster [DBG] pgmap v308: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 154 op/s 2024-07-28T19:33:14.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:14 smithi100 bash[31715]: cluster 2024-07-28T19:33:12.992567+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-07-28T19:33:15.043 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.043 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.044 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.044 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.044 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.044 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.045 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.045 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.045 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:15 smithi138 bash[23952]: cluster 2024-07-28T19:33:14.009275+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-07-28T19:33:15.045 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:15 smithi138 bash[23952]: audit 2024-07-28T19:33:14.192227+0000 mon.a (mon.0) 964 : audit [DBG] from='client.? 172.21.15.100:0/1822962920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:15.045 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:33:14 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:15.046 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:15.093 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:33:15.109 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 0 2024-07-28T19:33:15.140 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:33:15.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:15 smithi138 systemd[1]: Stopping Ceph mgr.x for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:33:15.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:15 smithi100 bash[21828]: cluster 2024-07-28T19:33:14.009275+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-07-28T19:33:15.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:15 smithi100 bash[21828]: audit 2024-07-28T19:33:14.192227+0000 mon.a (mon.0) 964 : audit [DBG] from='client.? 172.21.15.100:0/1822962920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:15.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:15 smithi100 bash[31715]: cluster 2024-07-28T19:33:14.009275+0000 mon.a (mon.0) 963 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-07-28T19:33:15.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:15 smithi100 bash[31715]: audit 2024-07-28T19:33:14.192227+0000 mon.a (mon.0) 964 : audit [DBG] from='client.? 172.21.15.100:0/1822962920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:16.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:16 smithi138 bash[23952]: cluster 2024-07-28T19:33:14.969298+0000 mgr.y (mgr.14391) 253 : cluster [DBG] pgmap v311: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 154 op/s 2024-07-28T19:33:16.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:16 smithi138 bash[23952]: cluster 2024-07-28T19:33:15.028453+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-07-28T19:33:16.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:16 smithi138 bash[23952]: audit 2024-07-28T19:33:15.913512+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:16.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:15 smithi138 bash[52287]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mgr-x 2024-07-28T19:33:16.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:15 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:33:16.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:15 smithi138 bash[52347]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mgr-x 2024-07-28T19:33:16.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:15 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-07-28T19:33:16.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:15 smithi138 systemd[1]: Stopped Ceph mgr.x for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:33:16.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:16 smithi100 bash[31715]: cluster 2024-07-28T19:33:14.969298+0000 mgr.y (mgr.14391) 253 : cluster [DBG] pgmap v311: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 154 op/s 2024-07-28T19:33:16.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:16 smithi100 bash[31715]: cluster 2024-07-28T19:33:15.028453+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-07-28T19:33:16.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:16 smithi100 bash[31715]: audit 2024-07-28T19:33:15.913512+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:16.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:16 smithi100 bash[21828]: cluster 2024-07-28T19:33:14.969298+0000 mgr.y (mgr.14391) 253 : cluster [DBG] pgmap v311: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 154 op/s 2024-07-28T19:33:16.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:16 smithi100 bash[21828]: cluster 2024-07-28T19:33:15.028453+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-07-28T19:33:16.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:16 smithi100 bash[21828]: audit 2024-07-28T19:33:15.913512+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:16.554 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:16.555 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:16.555 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:16.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:16.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:16.556 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:16.556 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:16.556 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: Started Ceph mgr.x for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:33:16.557 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:33:16 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:33:17.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:17 smithi138 bash[23952]: cluster 2024-07-28T19:33:16.041692+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-07-28T19:33:17.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:17 smithi138 bash[23952]: audit 2024-07-28T19:33:16.244001+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:17.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:17 smithi138 bash[23952]: audit 2024-07-28T19:33:16.258723+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:17.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:17 smithi138 bash[23952]: audit 2024-07-28T19:33:16.261219+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:17.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:17 smithi138 bash[23952]: audit 2024-07-28T19:33:16.541260+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.100:0/3864290315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:17.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[31715]: cluster 2024-07-28T19:33:16.041692+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-07-28T19:33:17.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[31715]: audit 2024-07-28T19:33:16.244001+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:17.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[31715]: audit 2024-07-28T19:33:16.258723+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:17.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[31715]: audit 2024-07-28T19:33:16.261219+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:17.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[31715]: audit 2024-07-28T19:33:16.541260+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.100:0/3864290315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:17.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[21828]: cluster 2024-07-28T19:33:16.041692+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-07-28T19:33:17.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[21828]: audit 2024-07-28T19:33:16.244001+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:17.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[21828]: audit 2024-07-28T19:33:16.258723+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:17.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[21828]: audit 2024-07-28T19:33:16.261219+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:17.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:17 smithi100 bash[21828]: audit 2024-07-28T19:33:16.541260+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.100:0/3864290315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:17.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:17 smithi138 bash[52402]: debug 2024-07-28T19:33:17.431+0000 7fb1fbc93640 1 -- 172.21.15.138:0/1369665813 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557888bf6d00 con 0x557888be8800 2024-07-28T19:33:17.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:17 smithi138 bash[52402]: debug 2024-07-28T19:33:17.551+0000 7fb1fe4fc040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:33:18.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:18 smithi138 bash[23952]: cluster 2024-07-28T19:33:16.970846+0000 mgr.y (mgr.14391) 254 : cluster [DBG] pgmap v314: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 4.0 MiB/s wr, 185 op/s 2024-07-28T19:33:18.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:18 smithi138 bash[23952]: cluster 2024-07-28T19:33:17.052278+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-07-28T19:33:18.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:18 smithi138 bash[23952]: cluster 2024-07-28T19:33:17.866018+0000 mon.a (mon.0) 970 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-07-28T19:33:18.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:18 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:18] "GET /metrics HTTP/1.1" 200 245808 "" "Prometheus/2.43.0" 2024-07-28T19:33:18.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:18 smithi100 bash[21828]: cluster 2024-07-28T19:33:16.970846+0000 mgr.y (mgr.14391) 254 : cluster [DBG] pgmap v314: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 4.0 MiB/s wr, 185 op/s 2024-07-28T19:33:18.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:18 smithi100 bash[21828]: cluster 2024-07-28T19:33:17.052278+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-07-28T19:33:18.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:18 smithi100 bash[21828]: cluster 2024-07-28T19:33:17.866018+0000 mon.a (mon.0) 970 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-07-28T19:33:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:18 smithi100 bash[31715]: cluster 2024-07-28T19:33:16.970846+0000 mgr.y (mgr.14391) 254 : cluster [DBG] pgmap v314: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 4.0 MiB/s wr, 185 op/s 2024-07-28T19:33:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:18 smithi100 bash[31715]: cluster 2024-07-28T19:33:17.052278+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-07-28T19:33:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:18 smithi100 bash[31715]: cluster 2024-07-28T19:33:17.866018+0000 mon.a (mon.0) 970 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-07-28T19:33:18.602 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:33:18.802 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:33:19.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:19 smithi138 bash[23952]: audit 2024-07-28T19:33:18.944712+0000 mon.c (mon.2) 159 : audit [DBG] from='client.? 172.21.15.100:0/98296127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:19.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:18 smithi138 bash[52402]: debug 2024-07-28T19:33:18.859+0000 7fb1fe4fc040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:33:19.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:18 smithi138 bash[52402]: debug 2024-07-28T19:33:18.923+0000 7fb1fe4fc040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:33:19.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:19 smithi138 bash[52402]: debug 2024-07-28T19:33:19.091+0000 7fb1fe4fc040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:33:19.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:19 smithi100 bash[21828]: audit 2024-07-28T19:33:18.944712+0000 mon.c (mon.2) 159 : audit [DBG] from='client.? 172.21.15.100:0/98296127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:19.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:19 smithi100 bash[31715]: audit 2024-07-28T19:33:18.944712+0000 mon.c (mon.2) 159 : audit [DBG] from='client.? 172.21.15.100:0/98296127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:20.051 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:19 smithi138 bash[52402]: debug 2024-07-28T19:33:19.747+0000 7fb1fe4fc040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:33:20.051 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:19 smithi138 bash[52402]: debug 2024-07-28T19:33:19.819+0000 7fb1fe4fc040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:33:20.051 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:19 smithi138 bash[52402]: debug 2024-07-28T19:33:19.887+0000 7fb1fe4fc040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:33:20.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[23952]: cluster 2024-07-28T19:33:18.971594+0000 mgr.y (mgr.14391) 255 : cluster [DBG] pgmap v317: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 4.0 MiB/s wr, 185 op/s 2024-07-28T19:33:20.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[23952]: cluster 2024-07-28T19:33:19.058444+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-07-28T19:33:20.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[52402]: debug 2024-07-28T19:33:20.087+0000 7fb1fe4fc040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:33:20.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[52402]: debug 2024-07-28T19:33:20.171+0000 7fb1fe4fc040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:33:20.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[52402]: debug 2024-07-28T19:33:20.239+0000 7fb1fe4fc040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:33:20.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:20 smithi100 bash[21828]: cluster 2024-07-28T19:33:18.971594+0000 mgr.y (mgr.14391) 255 : cluster [DBG] pgmap v317: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 4.0 MiB/s wr, 185 op/s 2024-07-28T19:33:20.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:20 smithi100 bash[21828]: cluster 2024-07-28T19:33:19.058444+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-07-28T19:33:20.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:20 smithi100 bash[31715]: cluster 2024-07-28T19:33:18.971594+0000 mgr.y (mgr.14391) 255 : cluster [DBG] pgmap v317: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 4.0 MiB/s wr, 185 op/s 2024-07-28T19:33:20.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:20 smithi100 bash[31715]: cluster 2024-07-28T19:33:19.058444+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-07-28T19:33:20.766 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[52402]: debug 2024-07-28T19:33:20.499+0000 7fb1fe4fc040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:33:20.767 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[52402]: debug 2024-07-28T19:33:20.571+0000 7fb1fe4fc040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:33:21.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[52402]: debug 2024-07-28T19:33:20.767+0000 7fb1fe4fc040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:33:21.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:20 smithi138 bash[52402]: debug 2024-07-28T19:33:20.847+0000 7fb1fe4fc040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:33:21.346 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (11890 ms) 2024-07-28T19:33:21.347 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-07-28T19:33:21.382 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:21.398 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:33:21.473 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:33:21.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:21 smithi100 bash[21828]: cluster 2024-07-28T19:33:20.071632+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-07-28T19:33:21.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:21 smithi100 bash[31715]: cluster 2024-07-28T19:33:20.071632+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-07-28T19:33:21.517 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:21 smithi138 bash[23952]: cluster 2024-07-28T19:33:20.071632+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-07-28T19:33:21.517 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:21 smithi138 bash[52402]: debug 2024-07-28T19:33:21.119+0000 7fb1fe4fc040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-28T19:33:21.517 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:21 smithi138 bash[52402]: debug 2024-07-28T19:33:21.319+0000 7fb1fe4fc040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:33:21.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:21 smithi138 bash[52402]: debug 2024-07-28T19:33:21.519+0000 7fb1fe4fc040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:33:21.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:21 smithi138 bash[52402]: debug 2024-07-28T19:33:21.607+0000 7fb1fe4fc040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:33:22.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:21 smithi138 bash[52402]: debug 2024-07-28T19:33:21.823+0000 7fb1fe4fc040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:33:22.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[23952]: cluster 2024-07-28T19:33:20.972319+0000 mgr.y (mgr.14391) 256 : cluster [DBG] pgmap v320: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:22.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[23952]: cluster 2024-07-28T19:33:21.076241+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-07-28T19:33:22.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[23952]: audit 2024-07-28T19:33:21.339139+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.100:0/4212895324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:22.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[23952]: audit 2024-07-28T19:33:21.635704+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[23952]: audit 2024-07-28T19:33:21.640437+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[23952]: audit 2024-07-28T19:33:22.017422+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[23952]: audit 2024-07-28T19:33:22.029938+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[21828]: cluster 2024-07-28T19:33:20.972319+0000 mgr.y (mgr.14391) 256 : cluster [DBG] pgmap v320: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[21828]: cluster 2024-07-28T19:33:21.076241+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-07-28T19:33:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[21828]: audit 2024-07-28T19:33:21.339139+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.100:0/4212895324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[21828]: audit 2024-07-28T19:33:21.635704+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[21828]: audit 2024-07-28T19:33:21.640437+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[21828]: audit 2024-07-28T19:33:22.017422+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[21828]: audit 2024-07-28T19:33:22.029938+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[31715]: cluster 2024-07-28T19:33:20.972319+0000 mgr.y (mgr.14391) 256 : cluster [DBG] pgmap v320: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:33:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[31715]: cluster 2024-07-28T19:33:21.076241+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-07-28T19:33:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[31715]: audit 2024-07-28T19:33:21.339139+0000 mon.c (mon.2) 160 : audit [DBG] from='client.? 172.21.15.100:0/4212895324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[31715]: audit 2024-07-28T19:33:21.635704+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[31715]: audit 2024-07-28T19:33:21.640437+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[31715]: audit 2024-07-28T19:33:22.017422+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:22 smithi100 bash[31715]: audit 2024-07-28T19:33:22.029938+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:23.014 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[52402]: debug 2024-07-28T19:33:22.707+0000 7fb1fe4fc040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:33:23.014 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:22 smithi138 bash[52402]: debug 2024-07-28T19:33:22.787+0000 7fb1fe4fc040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:33:23.109 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:33:23.147 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:33:23.147 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:33:23.147 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:33:23.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: debug 2024-07-28T19:33:23.015+0000 7fb1fe4fc040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:33:23.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: debug 2024-07-28T19:33:23.095+0000 7fb1fe4fc040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:33:23.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[23952]: cluster 2024-07-28T19:33:22.083629+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-07-28T19:33:23.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[23952]: audit 2024-07-28T19:33:22.601550+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:23.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[23952]: audit 2024-07-28T19:33:22.616825+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:23.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:23 smithi100 bash[31715]: cluster 2024-07-28T19:33:22.083629+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-07-28T19:33:23.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:23 smithi100 bash[31715]: audit 2024-07-28T19:33:22.601550+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:23.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:23 smithi100 bash[31715]: audit 2024-07-28T19:33:22.616825+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:23.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:23 smithi100 bash[21828]: cluster 2024-07-28T19:33:22.083629+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-07-28T19:33:23.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:23 smithi100 bash[21828]: audit 2024-07-28T19:33:22.601550+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:23.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:23 smithi100 bash[21828]: audit 2024-07-28T19:33:22.616825+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:23.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: debug 2024-07-28T19:33:23.519+0000 7fb1fe4fc040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:33:23.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: debug 2024-07-28T19:33:23.599+0000 7fb1fe4fc040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:33:23.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: [28/Jul/2024:19:33:23] ENGINE Bus STARTING 2024-07-28T19:33:23.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: CherryPy Checker: 2024-07-28T19:33:23.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: The Application mounted at '' has an empty config. 2024-07-28T19:33:23.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: [28/Jul/2024:19:33:23] ENGINE Serving on http://:::9283 2024-07-28T19:33:23.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:23 smithi138 bash[52402]: [28/Jul/2024:19:33:23] ENGINE Bus STARTED 2024-07-28T19:33:24.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: cluster 2024-07-28T19:33:22.973117+0000 mgr.y (mgr.14391) 257 : cluster [DBG] pgmap v323: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 471 op/s 2024-07-28T19:33:24.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: cluster 2024-07-28T19:33:23.089284+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-07-28T19:33:24.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: cluster 2024-07-28T19:33:23.624743+0000 mon.a (mon.0) 982 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:33:24.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: cluster 2024-07-28T19:33:23.624939+0000 mon.a (mon.0) 983 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:33:24.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: audit 2024-07-28T19:33:23.626035+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:33:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: audit 2024-07-28T19:33:23.626918+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:33:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: audit 2024-07-28T19:33:23.628702+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:33:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: audit 2024-07-28T19:33:23.629953+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:33:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[21828]: audit 2024-07-28T19:33:23.699496+0000 mon.a (mon.0) 984 : audit [DBG] from='client.? 172.21.15.100:0/3872126325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: cluster 2024-07-28T19:33:22.973117+0000 mgr.y (mgr.14391) 257 : cluster [DBG] pgmap v323: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 471 op/s 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: cluster 2024-07-28T19:33:23.089284+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: cluster 2024-07-28T19:33:23.624743+0000 mon.a (mon.0) 982 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: cluster 2024-07-28T19:33:23.624939+0000 mon.a (mon.0) 983 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: audit 2024-07-28T19:33:23.626035+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: audit 2024-07-28T19:33:23.626918+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: audit 2024-07-28T19:33:23.628702+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: audit 2024-07-28T19:33:23.629953+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:33:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:24 smithi100 bash[31715]: audit 2024-07-28T19:33:23.699496+0000 mon.a (mon.0) 984 : audit [DBG] from='client.? 172.21.15.100:0/3872126325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:24.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: cluster 2024-07-28T19:33:22.973117+0000 mgr.y (mgr.14391) 257 : cluster [DBG] pgmap v323: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 471 op/s 2024-07-28T19:33:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: cluster 2024-07-28T19:33:23.089284+0000 mon.a (mon.0) 981 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-07-28T19:33:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: cluster 2024-07-28T19:33:23.624743+0000 mon.a (mon.0) 982 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:33:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: cluster 2024-07-28T19:33:23.624939+0000 mon.a (mon.0) 983 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:33:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: audit 2024-07-28T19:33:23.626035+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:33:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: audit 2024-07-28T19:33:23.626918+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:33:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: audit 2024-07-28T19:33:23.628702+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:33:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: audit 2024-07-28T19:33:23.629953+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:33:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:24 smithi138 bash[23952]: audit 2024-07-28T19:33:23.699496+0000 mon.a (mon.0) 984 : audit [DBG] from='client.? 172.21.15.100:0/3872126325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:25.136 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:33:25.155 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:33:25.155 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:33:25.176 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:33:25.222 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:33:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:25 smithi100 bash[21828]: cluster 2024-07-28T19:33:24.117499+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-07-28T19:33:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:25 smithi100 bash[21828]: cluster 2024-07-28T19:33:24.630137+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e21: y(active, since 6m), standbys: x 2024-07-28T19:33:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:25 smithi100 bash[31715]: cluster 2024-07-28T19:33:24.117499+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-07-28T19:33:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:25 smithi100 bash[31715]: cluster 2024-07-28T19:33:24.630137+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e21: y(active, since 6m), standbys: x 2024-07-28T19:33:25.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:25 smithi138 bash[23952]: cluster 2024-07-28T19:33:24.117499+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-07-28T19:33:25.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:25 smithi138 bash[23952]: cluster 2024-07-28T19:33:24.630137+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e21: y(active, since 6m), standbys: x 2024-07-28T19:33:26.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:26 smithi100 bash[21828]: cluster 2024-07-28T19:33:24.973855+0000 mgr.y (mgr.14391) 258 : cluster [DBG] pgmap v326: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 471 op/s 2024-07-28T19:33:26.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:26 smithi100 bash[21828]: cluster 2024-07-28T19:33:25.116121+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-07-28T19:33:26.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:26 smithi100 bash[21828]: audit 2024-07-28T19:33:26.050131+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.100:0/407182968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:26.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:26 smithi100 bash[31715]: cluster 2024-07-28T19:33:24.973855+0000 mgr.y (mgr.14391) 258 : cluster [DBG] pgmap v326: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 471 op/s 2024-07-28T19:33:26.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:26 smithi100 bash[31715]: cluster 2024-07-28T19:33:25.116121+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-07-28T19:33:26.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:26 smithi100 bash[31715]: audit 2024-07-28T19:33:26.050131+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.100:0/407182968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:26.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:26 smithi138 bash[52402]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:33:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:26 smithi138 bash[23952]: cluster 2024-07-28T19:33:24.973855+0000 mgr.y (mgr.14391) 258 : cluster [DBG] pgmap v326: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 471 op/s 2024-07-28T19:33:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:26 smithi138 bash[23952]: cluster 2024-07-28T19:33:25.116121+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-07-28T19:33:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:26 smithi138 bash[23952]: audit 2024-07-28T19:33:26.050131+0000 mon.c (mon.2) 161 : audit [DBG] from='client.? 172.21.15.100:0/407182968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:27.168 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:27.222 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:33:27.245 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:33:27.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:27 smithi100 bash[21828]: cluster 2024-07-28T19:33:26.124041+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-07-28T19:33:27.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:27 smithi100 bash[31715]: cluster 2024-07-28T19:33:26.124041+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-07-28T19:33:27.501 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:27 smithi138 bash[23952]: cluster 2024-07-28T19:33:26.124041+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-07-28T19:33:28.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:28 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:28] "GET /metrics HTTP/1.1" 200 245808 "" "Prometheus/2.43.0" 2024-07-28T19:33:28.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:28 smithi100 bash[31715]: cluster 2024-07-28T19:33:26.975404+0000 mgr.y (mgr.14391) 259 : cluster [DBG] pgmap v329: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:28.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:28 smithi100 bash[31715]: cluster 2024-07-28T19:33:27.135602+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-07-28T19:33:28.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:28 smithi100 bash[31715]: cluster 2024-07-28T19:33:27.867446+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-07-28T19:33:28.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:28 smithi100 bash[21828]: cluster 2024-07-28T19:33:26.975404+0000 mgr.y (mgr.14391) 259 : cluster [DBG] pgmap v329: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:28.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:28 smithi100 bash[21828]: cluster 2024-07-28T19:33:27.135602+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-07-28T19:33:28.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:28 smithi100 bash[21828]: cluster 2024-07-28T19:33:27.867446+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-07-28T19:33:28.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:28 smithi138 bash[23952]: cluster 2024-07-28T19:33:26.975404+0000 mgr.y (mgr.14391) 259 : cluster [DBG] pgmap v329: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-07-28T19:33:28.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:28 smithi138 bash[23952]: cluster 2024-07-28T19:33:27.135602+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-07-28T19:33:28.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:28 smithi138 bash[23952]: cluster 2024-07-28T19:33:27.867446+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-07-28T19:33:29.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:29 smithi138 bash[23952]: audit 2024-07-28T19:33:28.398181+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.100:0/3775592622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:29.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:29 smithi138 bash[23952]: cluster 2024-07-28T19:33:28.870271+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-07-28T19:33:29.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:29 smithi100 bash[21828]: audit 2024-07-28T19:33:28.398181+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.100:0/3775592622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:29.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:29 smithi100 bash[21828]: cluster 2024-07-28T19:33:28.870271+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-07-28T19:33:29.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:29 smithi100 bash[31715]: audit 2024-07-28T19:33:28.398181+0000 mon.c (mon.2) 162 : audit [DBG] from='client.? 172.21.15.100:0/3775592622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:29.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:29 smithi100 bash[31715]: cluster 2024-07-28T19:33:28.870271+0000 mon.a (mon.0) 991 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-07-28T19:33:30.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:30 smithi138 bash[23952]: cluster 2024-07-28T19:33:28.976194+0000 mgr.y (mgr.14391) 260 : cluster [DBG] pgmap v333: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 73 KiB/s rd, 8.3 MiB/s wr, 107 op/s 2024-07-28T19:33:30.444 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:30 smithi138 bash[23952]: cluster 2024-07-28T19:33:29.878119+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-07-28T19:33:30.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:30 smithi100 bash[31715]: cluster 2024-07-28T19:33:28.976194+0000 mgr.y (mgr.14391) 260 : cluster [DBG] pgmap v333: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 73 KiB/s rd, 8.3 MiB/s wr, 107 op/s 2024-07-28T19:33:30.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:30 smithi100 bash[31715]: cluster 2024-07-28T19:33:29.878119+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-07-28T19:33:30.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:30 smithi100 bash[21828]: cluster 2024-07-28T19:33:28.976194+0000 mgr.y (mgr.14391) 260 : cluster [DBG] pgmap v333: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 73 KiB/s rd, 8.3 MiB/s wr, 107 op/s 2024-07-28T19:33:30.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:30 smithi100 bash[21828]: cluster 2024-07-28T19:33:29.878119+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-07-28T19:33:30.691 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 0 2024-07-28T19:33:30.797 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:33:30.930 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:33:31.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:31 smithi100 bash[21828]: audit 2024-07-28T19:33:30.794053+0000 mon.a (mon.0) 993 : audit [DBG] from='client.? 172.21.15.100:0/4050941074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:31.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:31 smithi100 bash[21828]: audit 2024-07-28T19:33:30.913355+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:31.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:31 smithi100 bash[31715]: audit 2024-07-28T19:33:30.794053+0000 mon.a (mon.0) 993 : audit [DBG] from='client.? 172.21.15.100:0/4050941074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:31.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:31 smithi100 bash[31715]: audit 2024-07-28T19:33:30.913355+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:31.533 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:31 smithi138 bash[23952]: audit 2024-07-28T19:33:30.794053+0000 mon.a (mon.0) 993 : audit [DBG] from='client.? 172.21.15.100:0/4050941074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:31.534 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:31 smithi138 bash[23952]: audit 2024-07-28T19:33:30.913355+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:32.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[21828]: cluster 2024-07-28T19:33:30.976963+0000 mgr.y (mgr.14391) 261 : cluster [DBG] pgmap v335: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:32.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[21828]: cluster 2024-07-28T19:33:31.162873+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-07-28T19:33:32.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[31715]: cluster 2024-07-28T19:33:30.976963+0000 mgr.y (mgr.14391) 261 : cluster [DBG] pgmap v335: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:32.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[31715]: cluster 2024-07-28T19:33:31.162873+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-07-28T19:33:32.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[22098]: debug 2024-07-28T19:33:32.320+0000 7fb35413e640 -1 mgr handle_mgr_map I was active but no longer am 2024-07-28T19:33:32.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[22098]: ignoring --setuser ceph since I am not root 2024-07-28T19:33:32.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[22098]: ignoring --setgroup ceph since I am not root 2024-07-28T19:33:32.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[22098]: debug 2024-07-28T19:33:32.380+0000 7f702bffa640 1 -- 172.21.15.100:0/3613781769 <== mon.2 v2:172.21.15.100:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a3a5e1e9c0 con 0x55a3a5e2b000 2024-07-28T19:33:32.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: [28/Jul/2024:19:33:32] ENGINE Bus STOPPING 2024-07-28T19:33:32.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[23952]: cluster 2024-07-28T19:33:30.976963+0000 mgr.y (mgr.14391) 261 : cluster [DBG] pgmap v335: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:32.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[23952]: cluster 2024-07-28T19:33:31.162873+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-07-28T19:33:32.688 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (11342 ms) 2024-07-28T19:33:32.688 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-07-28T19:33:32.766 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:32.792 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:33:32.886 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:33:32.903 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: [28/Jul/2024:19:33:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:33:32.903 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: [28/Jul/2024:19:33:32] ENGINE Bus STOPPED 2024-07-28T19:33:32.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:32 smithi100 bash[22098]: debug 2024-07-28T19:33:32.516+0000 7f702e854ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:33:33.163 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: [28/Jul/2024:19:33:32] ENGINE Bus STARTING 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.959+0000 7fb165b1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.959+0000 7fb165b1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.959+0000 7fb165b1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.959+0000 7fb165b1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.959+0000 7fb165b1b640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.967+0000 7fb16bb27640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.967+0000 7fb16bb27640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.967+0000 7fb16bb27640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.967+0000 7fb16bb27640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:32 smithi138 bash[52402]: debug 2024-07-28T19:33:32.967+0000 7fb16bb27640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:33:33.164 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[52402]: [28/Jul/2024:19:33:33] ENGINE Serving on http://:::9283 2024-07-28T19:33:33.165 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[52402]: [28/Jul/2024:19:33:33] ENGINE Bus STARTED 2024-07-28T19:33:33.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: cluster 2024-07-28T19:33:32.174748+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.189117+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.204461+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.207783+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.209610+0000 mon.c (mon.2) 165 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.220675+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.289422+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.301209+0000 mon.c (mon.2) 167 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.301861+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: cluster 2024-07-28T19:33:32.302911+0000 mon.a (mon.0) 1000 : cluster [INF] Activating manager daemon x 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: cluster 2024-07-28T19:33:32.320801+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.321754+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-07-28T19:33:33.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: cluster 2024-07-28T19:33:32.322032+0000 mon.a (mon.0) 1003 : cluster [DBG] mgrmap e22: x(active, starting, since 0.0191147s) 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.324010+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.324178+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.324326+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.325996+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.326350+0000 mon.a (mon.0) 1008 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.328418+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.328899+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.329225+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.329521+0000 mon.a (mon.0) 1012 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.329834+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.330135+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.330436+0000 mon.a (mon.0) 1015 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.330756+0000 mon.a (mon.0) 1016 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:33:33.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.331060+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.337953+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.338379+0000 mon.a (mon.0) 1019 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.339206+0000 mon.a (mon.0) 1020 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: cluster 2024-07-28T19:33:32.859994+0000 mon.a (mon.0) 1021 : cluster [INF] Manager daemon x is now available 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.884416+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.910602+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.916409+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:32.946837+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[31715]: audit 2024-07-28T19:33:33.148378+0000 mon.a (mon.0) 1026 : audit [DBG] from='client.? 172.21.15.100:0/1707789555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:33.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: cluster 2024-07-28T19:33:32.174748+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.189117+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.204461+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.207783+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.209610+0000 mon.c (mon.2) 165 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.220675+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.289422+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.301209+0000 mon.c (mon.2) 167 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.301861+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: cluster 2024-07-28T19:33:32.302911+0000 mon.a (mon.0) 1000 : cluster [INF] Activating manager daemon x 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: cluster 2024-07-28T19:33:32.320801+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.321754+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: cluster 2024-07-28T19:33:32.322032+0000 mon.a (mon.0) 1003 : cluster [DBG] mgrmap e22: x(active, starting, since 0.0191147s) 2024-07-28T19:33:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.324010+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.324178+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.324326+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.325996+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.326350+0000 mon.a (mon.0) 1008 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.328418+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.328899+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.329225+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.329521+0000 mon.a (mon.0) 1012 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.329834+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.330135+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.330436+0000 mon.a (mon.0) 1015 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.330756+0000 mon.a (mon.0) 1016 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.331060+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:33:33.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.337953+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:33:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.338379+0000 mon.a (mon.0) 1019 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:33:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.339206+0000 mon.a (mon.0) 1020 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:33:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: cluster 2024-07-28T19:33:32.859994+0000 mon.a (mon.0) 1021 : cluster [INF] Manager daemon x is now available 2024-07-28T19:33:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.884416+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.910602+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.916409+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:33:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:32.946837+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-28T19:33:33.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[21828]: audit 2024-07-28T19:33:33.148378+0000 mon.a (mon.0) 1026 : audit [DBG] from='client.? 172.21.15.100:0/1707789555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:33.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: cluster 2024-07-28T19:33:32.174748+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-07-28T19:33:33.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.189117+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.204461+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.207783+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.209610+0000 mon.c (mon.2) 165 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.220675+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14391 ' entity='mgr.y' 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.289422+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.301209+0000 mon.c (mon.2) 167 : audit [INF] from='mgr.14391 172.21.15.100:0/2310806595' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.301861+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: cluster 2024-07-28T19:33:32.302911+0000 mon.a (mon.0) 1000 : cluster [INF] Activating manager daemon x 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: cluster 2024-07-28T19:33:32.320801+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.321754+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.14391 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: cluster 2024-07-28T19:33:32.322032+0000 mon.a (mon.0) 1003 : cluster [DBG] mgrmap e22: x(active, starting, since 0.0191147s) 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.324010+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.324178+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.324326+0000 mon.a (mon.0) 1006 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:33:33.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.325996+0000 mon.a (mon.0) 1007 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.326350+0000 mon.a (mon.0) 1008 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.328418+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.328899+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.329225+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.329521+0000 mon.a (mon.0) 1012 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.329834+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.330135+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.330436+0000 mon.a (mon.0) 1015 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:33:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.330756+0000 mon.a (mon.0) 1016 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.331060+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.337953+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.338379+0000 mon.a (mon.0) 1019 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.339206+0000 mon.a (mon.0) 1020 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: cluster 2024-07-28T19:33:32.859994+0000 mon.a (mon.0) 1021 : cluster [INF] Manager daemon x is now available 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.884416+0000 mon.a (mon.0) 1022 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.910602+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.916409+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:32.946837+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-07-28T19:33:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:33 smithi138 bash[23952]: audit 2024-07-28T19:33:33.148378+0000 mon.a (mon.0) 1026 : audit [DBG] from='client.? 172.21.15.100:0/1707789555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:34.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[22098]: debug 2024-07-28T19:33:33.800+0000 7f702e854ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:33:34.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:33 smithi100 bash[22098]: debug 2024-07-28T19:33:33.864+0000 7f702e854ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:33:34.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[22098]: debug 2024-07-28T19:33:34.012+0000 7f702e854ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:33:34.357 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:33:34.387 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:33:34.387 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:33:34.387 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[21828]: cluster 2024-07-28T19:33:33.314653+0000 mon.a (mon.0) 1027 : cluster [DBG] mgrmap e23: x(active, since 1.01174s) 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[21828]: cluster 2024-07-28T19:33:33.326642+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[21828]: cephadm 2024-07-28T19:33:33.437288+0000 mgr.x (mgr.24484) 2 : cephadm [INF] Deploying cephadm binary to smithi138 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[21828]: audit 2024-07-28T19:33:33.612755+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[21828]: audit 2024-07-28T19:33:33.629705+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[31715]: cluster 2024-07-28T19:33:33.314653+0000 mon.a (mon.0) 1027 : cluster [DBG] mgrmap e23: x(active, since 1.01174s) 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[31715]: cluster 2024-07-28T19:33:33.326642+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[31715]: cephadm 2024-07-28T19:33:33.437288+0000 mgr.x (mgr.24484) 2 : cephadm [INF] Deploying cephadm binary to smithi138 2024-07-28T19:33:34.670 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[31715]: audit 2024-07-28T19:33:33.612755+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:34.671 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[31715]: audit 2024-07-28T19:33:33.629705+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:34.761 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[23952]: cluster 2024-07-28T19:33:33.314653+0000 mon.a (mon.0) 1027 : cluster [DBG] mgrmap e23: x(active, since 1.01174s) 2024-07-28T19:33:34.761 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[23952]: cluster 2024-07-28T19:33:33.326642+0000 mon.a (mon.0) 1028 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-07-28T19:33:34.761 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[23952]: cephadm 2024-07-28T19:33:33.437288+0000 mgr.x (mgr.24484) 2 : cephadm [INF] Deploying cephadm binary to smithi138 2024-07-28T19:33:34.761 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[23952]: audit 2024-07-28T19:33:33.612755+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:34.761 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[23952]: audit 2024-07-28T19:33:33.629705+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:34.961 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[22098]: debug 2024-07-28T19:33:34.672+0000 7f702e854ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:33:34.962 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[22098]: debug 2024-07-28T19:33:34.740+0000 7f702e854ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:33:34.962 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[22098]: debug 2024-07-28T19:33:34.804+0000 7f702e854ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:33:35.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: debug 2024-07-28T19:33:34.763+0000 7fb1522f4640 -1 log_channel(cephadm) log [ERR] : [28/Jul/2024:19:33:34] ENGINE Error in HTTPServer.serve 2024-07-28T19:33:35.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: Traceback (most recent call last): 2024-07-28T19:33:35.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:33:35.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: self._connections.run(self.expiration_interval) 2024-07-28T19:33:35.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:33:35.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: self._run(expiration_interval) 2024-07-28T19:33:35.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:33:35.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: s = self.context.wrap_socket( 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: return self.sslsocket_class._create( 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: self.do_handshake() 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: self._sslobj.do_handshake() 2024-07-28T19:33:35.056 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:34 smithi138 bash[52402]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:33:35.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:34 smithi100 bash[22098]: debug 2024-07-28T19:33:34.964+0000 7f702e854ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:33:35.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[22098]: debug 2024-07-28T19:33:35.040+0000 7f702e854ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:33:35.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[22098]: debug 2024-07-28T19:33:35.104+0000 7f702e854ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:33:35.651 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[21828]: cephadm 2024-07-28T19:33:33.468489+0000 mgr.x (mgr.24484) 3 : cephadm [INF] Deploying cephadm binary to smithi100 2024-07-28T19:33:35.651 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[21828]: cluster 2024-07-28T19:33:34.336259+0000 mgr.x (mgr.24484) 4 : cluster [DBG] pgmap v6: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:35.651 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[21828]: cluster 2024-07-28T19:33:34.338080+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-07-28T19:33:35.651 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[21828]: cluster 2024-07-28T19:33:34.634450+0000 mon.a (mon.0) 1032 : cluster [DBG] mgrmap e24: x(active, since 2s) 2024-07-28T19:33:35.652 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[22098]: debug 2024-07-28T19:33:35.336+0000 7f702e854ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:33:35.652 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[22098]: debug 2024-07-28T19:33:35.408+0000 7f702e854ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:33:35.652 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[22098]: debug 2024-07-28T19:33:35.580+0000 7f702e854ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:33:35.652 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[31715]: cephadm 2024-07-28T19:33:33.468489+0000 mgr.x (mgr.24484) 3 : cephadm [INF] Deploying cephadm binary to smithi100 2024-07-28T19:33:35.653 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[31715]: cluster 2024-07-28T19:33:34.336259+0000 mgr.x (mgr.24484) 4 : cluster [DBG] pgmap v6: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:35.653 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[31715]: cluster 2024-07-28T19:33:34.338080+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-07-28T19:33:35.653 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[31715]: cluster 2024-07-28T19:33:34.634450+0000 mon.a (mon.0) 1032 : cluster [DBG] mgrmap e24: x(active, since 2s) 2024-07-28T19:33:35.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:35 smithi138 bash[23952]: cephadm 2024-07-28T19:33:33.468489+0000 mgr.x (mgr.24484) 3 : cephadm [INF] Deploying cephadm binary to smithi100 2024-07-28T19:33:35.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:35 smithi138 bash[23952]: cluster 2024-07-28T19:33:34.336259+0000 mgr.x (mgr.24484) 4 : cluster [DBG] pgmap v6: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:35.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:35 smithi138 bash[23952]: cluster 2024-07-28T19:33:34.338080+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-07-28T19:33:35.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:35 smithi138 bash[23952]: cluster 2024-07-28T19:33:34.634450+0000 mon.a (mon.0) 1032 : cluster [DBG] mgrmap e24: x(active, since 2s) 2024-07-28T19:33:35.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[22098]: debug 2024-07-28T19:33:35.652+0000 7f702e854ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:33:35.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:35 smithi100 bash[22098]: debug 2024-07-28T19:33:35.884+0000 7f702e854ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:33:36.353 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[22098]: debug 2024-07-28T19:33:36.040+0000 7f702e854ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:33:36.353 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[22098]: debug 2024-07-28T19:33:36.112+0000 7f702e854ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:33:36.353 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[22098]: debug 2024-07-28T19:33:36.264+0000 7f702e854ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:33:36.379 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:33:36.393 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:33:36.393 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:33:36.413 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:33:36.455 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:33:36.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[52402]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:36] "GET /metrics HTTP/1.1" 200 35790 "" "Prometheus/2.43.0" 2024-07-28T19:33:36.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: cephadm 2024-07-28T19:33:34.643082+0000 mgr.x (mgr.24484) 5 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Bus STARTING 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: cephadm 2024-07-28T19:33:34.762331+0000 mgr.x (mgr.24484) 6 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Serving on https://172.21.15.138:7150 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: cephadm 2024-07-28T19:33:34.765809+0000 mgr.x (mgr.24484) 7 : cephadm [ERR] [28/Jul/2024:19:33:34] ENGINE Error in HTTPServer.serve 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: Traceback (most recent call last): 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: self._connections.run(self.expiration_interval) 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: self._run(expiration_interval) 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:33:36.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: s = self.context.wrap_socket( 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: return self.sslsocket_class._create( 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: self.do_handshake() 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: self._sslobj.do_handshake() 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: cephadm 2024-07-28T19:33:34.863513+0000 mgr.x (mgr.24484) 8 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Serving on http://172.21.15.138:8765 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: cephadm 2024-07-28T19:33:34.863651+0000 mgr.x (mgr.24484) 9 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Bus STARTED 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: cluster 2024-07-28T19:33:35.346486+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-07-28T19:33:36.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:36 smithi138 bash[23952]: audit 2024-07-28T19:33:35.503101+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.100:0/1526214306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:36.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: cephadm 2024-07-28T19:33:34.643082+0000 mgr.x (mgr.24484) 5 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Bus STARTING 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: cephadm 2024-07-28T19:33:34.762331+0000 mgr.x (mgr.24484) 6 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Serving on https://172.21.15.138:7150 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: cephadm 2024-07-28T19:33:34.765809+0000 mgr.x (mgr.24484) 7 : cephadm [ERR] [28/Jul/2024:19:33:34] ENGINE Error in HTTPServer.serve 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: Traceback (most recent call last): 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: self._connections.run(self.expiration_interval) 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: self._run(expiration_interval) 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:33:36.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: s = self.context.wrap_socket( 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: return self.sslsocket_class._create( 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: self.do_handshake() 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: self._sslobj.do_handshake() 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: cephadm 2024-07-28T19:33:34.863513+0000 mgr.x (mgr.24484) 8 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Serving on http://172.21.15.138:8765 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: cephadm 2024-07-28T19:33:34.863651+0000 mgr.x (mgr.24484) 9 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Bus STARTED 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: cluster 2024-07-28T19:33:35.346486+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-07-28T19:33:36.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[31715]: audit 2024-07-28T19:33:35.503101+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.100:0/1526214306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: cephadm 2024-07-28T19:33:34.643082+0000 mgr.x (mgr.24484) 5 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Bus STARTING 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: cephadm 2024-07-28T19:33:34.762331+0000 mgr.x (mgr.24484) 6 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Serving on https://172.21.15.138:7150 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: cephadm 2024-07-28T19:33:34.765809+0000 mgr.x (mgr.24484) 7 : cephadm [ERR] [28/Jul/2024:19:33:34] ENGINE Error in HTTPServer.serve 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: Traceback (most recent call last): 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: self._connections.run(self.expiration_interval) 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: self._run(expiration_interval) 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:33:36.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: s = self.context.wrap_socket( 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: return self.sslsocket_class._create( 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: self.do_handshake() 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: self._sslobj.do_handshake() 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: cephadm 2024-07-28T19:33:34.863513+0000 mgr.x (mgr.24484) 8 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Serving on http://172.21.15.138:8765 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: cephadm 2024-07-28T19:33:34.863651+0000 mgr.x (mgr.24484) 9 : cephadm [INF] [28/Jul/2024:19:33:34] ENGINE Bus STARTED 2024-07-28T19:33:36.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: cluster 2024-07-28T19:33:35.346486+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-07-28T19:33:36.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:36 smithi100 bash[21828]: audit 2024-07-28T19:33:35.503101+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.100:0/1526214306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:37.338 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: debug 2024-07-28T19:33:37.076+0000 7f702e854ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:33:37.338 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: debug 2024-07-28T19:33:37.148+0000 7f702e854ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:33:37.739 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[31715]: cluster 2024-07-28T19:33:36.337061+0000 mgr.x (mgr.24484) 10 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:37.739 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[31715]: cluster 2024-07-28T19:33:36.353120+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-07-28T19:33:37.739 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[31715]: cluster 2024-07-28T19:33:36.376813+0000 mon.a (mon.0) 1035 : cluster [DBG] mgrmap e25: x(active, since 4s) 2024-07-28T19:33:37.739 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[21828]: cluster 2024-07-28T19:33:36.337061+0000 mgr.x (mgr.24484) 10 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:37.739 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[21828]: cluster 2024-07-28T19:33:36.353120+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-07-28T19:33:37.739 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[21828]: cluster 2024-07-28T19:33:36.376813+0000 mon.a (mon.0) 1035 : cluster [DBG] mgrmap e25: x(active, since 4s) 2024-07-28T19:33:37.741 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: debug 2024-07-28T19:33:37.340+0000 7f702e854ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:33:37.741 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: debug 2024-07-28T19:33:37.416+0000 7f702e854ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:33:37.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:37 smithi138 bash[23952]: cluster 2024-07-28T19:33:36.337061+0000 mgr.x (mgr.24484) 10 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:37.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:37 smithi138 bash[23952]: cluster 2024-07-28T19:33:36.353120+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-07-28T19:33:37.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:37 smithi138 bash[23952]: cluster 2024-07-28T19:33:36.376813+0000 mon.a (mon.0) 1035 : cluster [DBG] mgrmap e25: x(active, since 4s) 2024-07-28T19:33:37.888 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:37.928 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:33:37.942 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:33:37.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: debug 2024-07-28T19:33:37.744+0000 7f702e854ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:33:37.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: debug 2024-07-28T19:33:37.820+0000 7f702e854ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:33:37.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: [28/Jul/2024:19:33:37] ENGINE Bus STARTING 2024-07-28T19:33:37.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: CherryPy Checker: 2024-07-28T19:33:37.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: The Application mounted at '' has an empty config. 2024-07-28T19:33:37.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: [28/Jul/2024:19:33:37] ENGINE Serving on http://:::9283 2024-07-28T19:33:37.994 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:37 smithi100 bash[22098]: [28/Jul/2024:19:33:37] ENGINE Bus STARTED 2024-07-28T19:33:38.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:33:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[31715]: cluster 2024-07-28T19:33:37.401614+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-07-28T19:33:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[31715]: cluster 2024-07-28T19:33:37.844190+0000 mon.a (mon.0) 1037 : cluster [DBG] Standby manager daemon y started 2024-07-28T19:33:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[31715]: audit 2024-07-28T19:33:37.844752+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-28T19:33:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[31715]: audit 2024-07-28T19:33:37.847755+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:33:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[31715]: audit 2024-07-28T19:33:37.848941+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.100:0/3716737077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[31715]: audit 2024-07-28T19:33:37.849286+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[31715]: audit 2024-07-28T19:33:37.849998+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[31715]: cluster 2024-07-28T19:33:37.866860+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[21828]: cluster 2024-07-28T19:33:37.401614+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[21828]: cluster 2024-07-28T19:33:37.844190+0000 mon.a (mon.0) 1037 : cluster [DBG] Standby manager daemon y started 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[21828]: audit 2024-07-28T19:33:37.844752+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[21828]: audit 2024-07-28T19:33:37.847755+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[21828]: audit 2024-07-28T19:33:37.848941+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.100:0/3716737077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[21828]: audit 2024-07-28T19:33:37.849286+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[21828]: audit 2024-07-28T19:33:37.849998+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:33:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:38 smithi100 bash[21828]: cluster 2024-07-28T19:33:37.866860+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-07-28T19:33:38.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:38 smithi138 bash[23952]: cluster 2024-07-28T19:33:37.401614+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-07-28T19:33:38.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:38 smithi138 bash[23952]: cluster 2024-07-28T19:33:37.844190+0000 mon.a (mon.0) 1037 : cluster [DBG] Standby manager daemon y started 2024-07-28T19:33:38.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:38 smithi138 bash[23952]: audit 2024-07-28T19:33:37.844752+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-28T19:33:38.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:38 smithi138 bash[23952]: audit 2024-07-28T19:33:37.847755+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:33:38.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:38 smithi138 bash[23952]: audit 2024-07-28T19:33:37.848941+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.100:0/3716737077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:38.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:38 smithi138 bash[23952]: audit 2024-07-28T19:33:37.849286+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-28T19:33:38.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:38 smithi138 bash[23952]: audit 2024-07-28T19:33:37.849998+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.100:0/1756849897' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:33:38.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:38 smithi138 bash[23952]: cluster 2024-07-28T19:33:37.866860+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-07-28T19:33:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:39 smithi100 bash[21828]: cluster 2024-07-28T19:33:38.337962+0000 mgr.x (mgr.24484) 11 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:39 smithi100 bash[21828]: cluster 2024-07-28T19:33:38.416264+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-07-28T19:33:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:39 smithi100 bash[21828]: audit 2024-07-28T19:33:38.417772+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:33:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:39 smithi100 bash[21828]: cluster 2024-07-28T19:33:38.881781+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-07-28T19:33:39.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:39 smithi100 bash[31715]: cluster 2024-07-28T19:33:38.337962+0000 mgr.x (mgr.24484) 11 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:39.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:39 smithi100 bash[31715]: cluster 2024-07-28T19:33:38.416264+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-07-28T19:33:39.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:39 smithi100 bash[31715]: audit 2024-07-28T19:33:38.417772+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:33:39.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:39 smithi100 bash[31715]: cluster 2024-07-28T19:33:38.881781+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-07-28T19:33:39.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:39 smithi138 bash[23952]: cluster 2024-07-28T19:33:38.337962+0000 mgr.x (mgr.24484) 11 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:39.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:39 smithi138 bash[23952]: cluster 2024-07-28T19:33:38.416264+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e26: x(active, since 6s), standbys: y 2024-07-28T19:33:39.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:39 smithi138 bash[23952]: audit 2024-07-28T19:33:38.417772+0000 mon.a (mon.0) 1040 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:33:39.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:39 smithi138 bash[23952]: cluster 2024-07-28T19:33:38.881781+0000 mon.a (mon.0) 1041 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-07-28T19:33:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:39.530299+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:39.536926+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: cluster 2024-07-28T19:33:39.882214+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-07-28T19:33:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:39.985982+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:39.990764+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:40.195798+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.100:0/1019944665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:40.253427+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:40.258403+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:40.259629+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:40.260494+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: cephadm 2024-07-28T19:33:40.260678+0000 mgr.x (mgr.24484) 12 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 2524M 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:40.261143+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:40.261783+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: audit 2024-07-28T19:33:40.267490+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:40 smithi138 bash[23952]: cluster 2024-07-28T19:33:40.338574+0000 mgr.x (mgr.24484) 13 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 192 KiB/s rd, 8.0 MiB/s wr, 231 op/s 2024-07-28T19:33:40.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:39.530299+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:39.536926+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: cluster 2024-07-28T19:33:39.882214+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-07-28T19:33:40.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:39.985982+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:39.990764+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:40.195798+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.100:0/1019944665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:40.253427+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:40.258403+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:40.259629+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:40.260494+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: cephadm 2024-07-28T19:33:40.260678+0000 mgr.x (mgr.24484) 12 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 2524M 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:40.261143+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:40.261783+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: audit 2024-07-28T19:33:40.267490+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[31715]: cluster 2024-07-28T19:33:40.338574+0000 mgr.x (mgr.24484) 13 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 192 KiB/s rd, 8.0 MiB/s wr, 231 op/s 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:39.530299+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:39.536926+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: cluster 2024-07-28T19:33:39.882214+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:39.985982+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:39.990764+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:40.195798+0000 mon.c (mon.2) 174 : audit [DBG] from='client.? 172.21.15.100:0/1019944665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:40.253427+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:40.258403+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:40.259629+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:40.260494+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: cephadm 2024-07-28T19:33:40.260678+0000 mgr.x (mgr.24484) 12 : cephadm [INF] Adjusting osd_memory_target on smithi100 to 2524M 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:40.261143+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:40.261783+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:40.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: audit 2024-07-28T19:33:40.267490+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:40.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:40 smithi100 bash[21828]: cluster 2024-07-28T19:33:40.338574+0000 mgr.x (mgr.24484) 13 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 192 KiB/s rd, 8.0 MiB/s wr, 231 op/s 2024-07-28T19:33:41.762 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:33:41.791 INFO:tasks.workunit.client.0.smithi100.stdout:write: * 83886080~1000 2024-07-28T19:33:41.858 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:33:41.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:41 smithi100 bash[31715]: audit 2024-07-28T19:33:40.695708+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:41.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:41 smithi100 bash[31715]: audit 2024-07-28T19:33:40.704981+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:41.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:41 smithi100 bash[31715]: cluster 2024-07-28T19:33:40.884595+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-07-28T19:33:41.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:41 smithi100 bash[21828]: audit 2024-07-28T19:33:40.695708+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:41.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:41 smithi100 bash[21828]: audit 2024-07-28T19:33:40.704981+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:41.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:41 smithi100 bash[21828]: cluster 2024-07-28T19:33:40.884595+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-07-28T19:33:42.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:41 smithi138 bash[23952]: audit 2024-07-28T19:33:40.695708+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:42.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:41 smithi138 bash[23952]: audit 2024-07-28T19:33:40.704981+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:42.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:41 smithi138 bash[23952]: cluster 2024-07-28T19:33:40.884595+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-07-28T19:33:42.100 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:33:42.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:42 smithi100 bash[21828]: cluster 2024-07-28T19:33:42.339326+0000 mgr.x (mgr.24484) 14 : cluster [DBG] pgmap v17: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 171 KiB/s rd, 7.2 MiB/s wr, 206 op/s 2024-07-28T19:33:42.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:42 smithi100 bash[21828]: audit 2024-07-28T19:33:42.552197+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.100:0/3738045546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:42.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:42 smithi100 bash[31715]: cluster 2024-07-28T19:33:42.339326+0000 mgr.x (mgr.24484) 14 : cluster [DBG] pgmap v17: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 171 KiB/s rd, 7.2 MiB/s wr, 206 op/s 2024-07-28T19:33:42.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:42 smithi100 bash[31715]: audit 2024-07-28T19:33:42.552197+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.100:0/3738045546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:43.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:42 smithi138 bash[23952]: cluster 2024-07-28T19:33:42.339326+0000 mgr.x (mgr.24484) 14 : cluster [DBG] pgmap v17: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 171 KiB/s rd, 7.2 MiB/s wr, 206 op/s 2024-07-28T19:33:43.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:42 smithi138 bash[23952]: audit 2024-07-28T19:33:42.552197+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.100:0/3738045546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:43.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:43 smithi100 bash[21828]: cluster 2024-07-28T19:33:42.713458+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-07-28T19:33:43.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:43 smithi100 bash[31715]: cluster 2024-07-28T19:33:42.713458+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-07-28T19:33:44.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:43 smithi138 bash[23952]: cluster 2024-07-28T19:33:42.713458+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-07-28T19:33:44.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:44 smithi100 bash[21828]: cluster 2024-07-28T19:33:43.713094+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-07-28T19:33:44.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:44 smithi100 bash[21828]: cluster 2024-07-28T19:33:44.340151+0000 mgr.x (mgr.24484) 15 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.4 MiB/s wr, 380 op/s 2024-07-28T19:33:44.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:44 smithi100 bash[31715]: cluster 2024-07-28T19:33:43.713094+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-07-28T19:33:44.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:44 smithi100 bash[31715]: cluster 2024-07-28T19:33:44.340151+0000 mgr.x (mgr.24484) 15 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.4 MiB/s wr, 380 op/s 2024-07-28T19:33:45.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:44 smithi138 bash[23952]: cluster 2024-07-28T19:33:43.713094+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-07-28T19:33:45.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:44 smithi138 bash[23952]: cluster 2024-07-28T19:33:44.340151+0000 mgr.x (mgr.24484) 15 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.4 MiB/s wr, 380 op/s 2024-07-28T19:33:45.327 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (12639 ms) 2024-07-28T19:33:45.327 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-07-28T19:33:45.411 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:45.440 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 41943040~10 2024-07-28T19:33:45.464 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:33:45.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:45 smithi100 bash[21828]: cluster 2024-07-28T19:33:44.739007+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-07-28T19:33:45.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:45 smithi100 bash[21828]: audit 2024-07-28T19:33:44.934528+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.100:0/3875124775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:45.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:45 smithi100 bash[31715]: cluster 2024-07-28T19:33:44.739007+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-07-28T19:33:45.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:45 smithi100 bash[31715]: audit 2024-07-28T19:33:44.934528+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.100:0/3875124775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:46.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:45 smithi138 bash[23952]: cluster 2024-07-28T19:33:44.739007+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-07-28T19:33:46.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:45 smithi138 bash[23952]: audit 2024-07-28T19:33:44.934528+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.100:0/3875124775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:46.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:46 smithi138 bash[52402]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:46] "GET /metrics HTTP/1.1" 200 35790 "" "Prometheus/2.43.0" 2024-07-28T19:33:46.768 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 41943040 2024-07-28T19:33:46.821 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:33:47.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:46 smithi138 bash[23952]: cluster 2024-07-28T19:33:45.743484+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-07-28T19:33:47.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:46 smithi138 bash[23952]: cluster 2024-07-28T19:33:46.340915+0000 mgr.x (mgr.24484) 16 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 423 op/s 2024-07-28T19:33:47.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:46 smithi100 bash[31715]: cluster 2024-07-28T19:33:45.743484+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-07-28T19:33:47.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:46 smithi100 bash[31715]: cluster 2024-07-28T19:33:46.340915+0000 mgr.x (mgr.24484) 16 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 423 op/s 2024-07-28T19:33:47.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:46 smithi100 bash[21828]: cluster 2024-07-28T19:33:45.743484+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-07-28T19:33:47.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:46 smithi100 bash[21828]: cluster 2024-07-28T19:33:46.340915+0000 mgr.x (mgr.24484) 16 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 423 op/s 2024-07-28T19:33:48.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:47 smithi138 bash[23952]: cluster 2024-07-28T19:33:46.749253+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-07-28T19:33:48.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:47 smithi138 bash[23952]: audit 2024-07-28T19:33:47.299130+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.100:0/1125867355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:48.159 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:48.177 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:47 smithi100 bash[31715]: cluster 2024-07-28T19:33:46.749253+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-07-28T19:33:48.177 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:47 smithi100 bash[31715]: audit 2024-07-28T19:33:47.299130+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.100:0/1125867355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:48.177 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:47 smithi100 bash[21828]: cluster 2024-07-28T19:33:46.749253+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-07-28T19:33:48.177 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:47 smithi100 bash[21828]: audit 2024-07-28T19:33:47.299130+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.100:0/1125867355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:48.187 INFO:tasks.workunit.client.0.smithi100.stdout:write: Y 41943040~5 2024-07-28T19:33:48.430 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:33:48.493 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:33:48.641 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:33:49.050 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:48 smithi138 bash[23952]: cluster 2024-07-28T19:33:47.773288+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-07-28T19:33:49.050 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:48 smithi138 bash[23952]: cluster 2024-07-28T19:33:47.879781+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-07-28T19:33:49.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:48 smithi138 bash[23952]: audit 2024-07-28T19:33:47.909360+0000 mon.a (mon.0) 1064 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:49.051 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:48 smithi138 bash[23952]: cluster 2024-07-28T19:33:48.341707+0000 mgr.x (mgr.24484) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:49.074 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (3747 ms) 2024-07-28T19:33:49.075 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-07-28T19:33:49.147 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:49.173 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:33:49.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[21828]: cluster 2024-07-28T19:33:47.773288+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-07-28T19:33:49.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[21828]: cluster 2024-07-28T19:33:47.879781+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-07-28T19:33:49.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[21828]: audit 2024-07-28T19:33:47.909360+0000 mon.a (mon.0) 1064 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:49.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[21828]: cluster 2024-07-28T19:33:48.341707+0000 mgr.x (mgr.24484) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:49.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[31715]: cluster 2024-07-28T19:33:47.773288+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-07-28T19:33:49.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[31715]: cluster 2024-07-28T19:33:47.879781+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-07-28T19:33:49.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[31715]: audit 2024-07-28T19:33:47.909360+0000 mon.a (mon.0) 1064 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:33:49.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:48 smithi100 bash[31715]: cluster 2024-07-28T19:33:48.341707+0000 mgr.x (mgr.24484) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:49.274 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:33:50.229 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:49 smithi138 bash[23952]: cluster 2024-07-28T19:33:48.885228+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-07-28T19:33:50.230 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:49 smithi138 bash[23952]: audit 2024-07-28T19:33:49.656057+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.100:0/2954714972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:50.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:49 smithi100 bash[21828]: cluster 2024-07-28T19:33:48.885228+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-07-28T19:33:50.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:49 smithi100 bash[21828]: audit 2024-07-28T19:33:49.656057+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.100:0/2954714972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:50.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:49 smithi100 bash[31715]: cluster 2024-07-28T19:33:48.885228+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-07-28T19:33:50.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:49 smithi100 bash[31715]: audit 2024-07-28T19:33:49.656057+0000 mon.a (mon.0) 1066 : audit [DBG] from='client.? 172.21.15.100:0/2954714972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:50.916 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:33:50.937 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:33:50.937 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:33:50.937 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:33:51.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: cluster 2024-07-28T19:33:49.888842+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-07-28T19:33:51.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: cluster 2024-07-28T19:33:50.342517+0000 mgr.x (mgr.24484) 18 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 22 KiB/s wr, 333 op/s 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.797447+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.807110+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.809407+0000 mon.a (mon.0) 1070 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.811339+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.813168+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.815313+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.825113+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.827099+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[21828]: audit 2024-07-28T19:33:50.828576+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: cluster 2024-07-28T19:33:49.888842+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: cluster 2024-07-28T19:33:50.342517+0000 mgr.x (mgr.24484) 18 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 22 KiB/s wr, 333 op/s 2024-07-28T19:33:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.797447+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.807110+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.809407+0000 mon.a (mon.0) 1070 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.811339+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.813168+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.815313+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.825113+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.827099+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:51.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:50 smithi100 bash[31715]: audit 2024-07-28T19:33:50.828576+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:33:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: cluster 2024-07-28T19:33:49.888842+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-07-28T19:33:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: cluster 2024-07-28T19:33:50.342517+0000 mgr.x (mgr.24484) 18 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 36 MiB/s rd, 22 KiB/s wr, 333 op/s 2024-07-28T19:33:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.797447+0000 mon.a (mon.0) 1068 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.807110+0000 mon.a (mon.0) 1069 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.809407+0000 mon.a (mon.0) 1070 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.811339+0000 mon.a (mon.0) 1071 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.813168+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.815313+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-07-28T19:33:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.825113+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.827099+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:33:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:50 smithi138 bash[23952]: audit 2024-07-28T19:33:50.828576+0000 mon.a (mon.0) 1076 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:33:52.123 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:51 smithi100 systemd[1]: Stopping Ceph node-exporter.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:33:52.123 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:50.815316+0000 mgr.x (mgr.24484) 19 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 2524M 2024-07-28T19:33:52.123 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:50.829381+0000 mgr.x (mgr.24484) 20 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:33:52.123 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:50.829811+0000 mgr.x (mgr.24484) 21 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:33:52.123 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cluster 2024-07-28T19:33:50.905789+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-07-28T19:33:52.123 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:51.002379+0000 mgr.x (mgr.24484) 22 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:51.010342+0000 mgr.x (mgr.24484) 23 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:51.122299+0000 mgr.x (mgr.24484) 24 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:51.142985+0000 mgr.x (mgr.24484) 25 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:51.248497+0000 mgr.x (mgr.24484) 26 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: cephadm 2024-07-28T19:33:51.284868+0000 mgr.x (mgr.24484) 27 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.378491+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.388405+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.422020+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.428840+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.434021+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.480515+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.486827+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.124 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.492852+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[21828]: audit 2024-07-28T19:33:51.498953+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:50.815316+0000 mgr.x (mgr.24484) 19 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 2524M 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:50.829381+0000 mgr.x (mgr.24484) 20 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:50.829811+0000 mgr.x (mgr.24484) 21 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cluster 2024-07-28T19:33:50.905789+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:51.002379+0000 mgr.x (mgr.24484) 22 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:51.010342+0000 mgr.x (mgr.24484) 23 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:51.122299+0000 mgr.x (mgr.24484) 24 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:51.142985+0000 mgr.x (mgr.24484) 25 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:33:52.125 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:51.248497+0000 mgr.x (mgr.24484) 26 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: cephadm 2024-07-28T19:33:51.284868+0000 mgr.x (mgr.24484) 27 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.378491+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.388405+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.422020+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.428840+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.434021+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.480515+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.486827+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.492852+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.126 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:51 smithi100 bash[31715]: audit 2024-07-28T19:33:51.498953+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:50.815316+0000 mgr.x (mgr.24484) 19 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 2524M 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:50.829381+0000 mgr.x (mgr.24484) 20 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:50.829811+0000 mgr.x (mgr.24484) 21 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cluster 2024-07-28T19:33:50.905789+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:51.002379+0000 mgr.x (mgr.24484) 22 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:51.010342+0000 mgr.x (mgr.24484) 23 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:51.122299+0000 mgr.x (mgr.24484) 24 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:51.142985+0000 mgr.x (mgr.24484) 25 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:51.248497+0000 mgr.x (mgr.24484) 26 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: cephadm 2024-07-28T19:33:51.284868+0000 mgr.x (mgr.24484) 27 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.378491+0000 mon.a (mon.0) 1078 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.388405+0000 mon.a (mon.0) 1079 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.422020+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.428840+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.434021+0000 mon.a (mon.0) 1082 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.480515+0000 mon.a (mon.0) 1083 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.486827+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.492852+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:51 smithi138 bash[23952]: audit 2024-07-28T19:33:51.498953+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:52.907 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:33:52.928 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:33:52.929 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:33:52.952 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:33:52.996 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:33:53.243 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:53 smithi100 bash[72225]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-node-exporter-a 2024-07-28T19:33:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[31715]: cephadm 2024-07-28T19:33:51.498907+0000 mgr.x (mgr.24484) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-07-28T19:33:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[31715]: cephadm 2024-07-28T19:33:51.499353+0000 mgr.x (mgr.24484) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi100 2024-07-28T19:33:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[31715]: cluster 2024-07-28T19:33:51.912169+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-07-28T19:33:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[31715]: audit 2024-07-28T19:33:52.023271+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.100:0/449440083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[31715]: cluster 2024-07-28T19:33:52.343322+0000 mgr.x (mgr.24484) 30 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 19 KiB/s wr, 300 op/s 2024-07-28T19:33:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[31715]: cluster 2024-07-28T19:33:52.880248+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-07-28T19:33:53.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[21828]: cephadm 2024-07-28T19:33:51.498907+0000 mgr.x (mgr.24484) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-07-28T19:33:53.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[21828]: cephadm 2024-07-28T19:33:51.499353+0000 mgr.x (mgr.24484) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi100 2024-07-28T19:33:53.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[21828]: cluster 2024-07-28T19:33:51.912169+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-07-28T19:33:53.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[21828]: audit 2024-07-28T19:33:52.023271+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.100:0/449440083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:53.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[21828]: cluster 2024-07-28T19:33:52.343322+0000 mgr.x (mgr.24484) 30 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 19 KiB/s wr, 300 op/s 2024-07-28T19:33:53.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:52 smithi100 bash[21828]: cluster 2024-07-28T19:33:52.880248+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-07-28T19:33:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:52 smithi138 bash[23952]: cephadm 2024-07-28T19:33:51.498907+0000 mgr.x (mgr.24484) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-07-28T19:33:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:52 smithi138 bash[23952]: cephadm 2024-07-28T19:33:51.499353+0000 mgr.x (mgr.24484) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi100 2024-07-28T19:33:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:52 smithi138 bash[23952]: cluster 2024-07-28T19:33:51.912169+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-07-28T19:33:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:52 smithi138 bash[23952]: audit 2024-07-28T19:33:52.023271+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.100:0/449440083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:52 smithi138 bash[23952]: cluster 2024-07-28T19:33:52.343322+0000 mgr.x (mgr.24484) 30 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 19 KiB/s wr, 300 op/s 2024-07-28T19:33:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:52 smithi138 bash[23952]: cluster 2024-07-28T19:33:52.880248+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-07-28T19:33:53.744 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:53 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:33:53.744 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:53 smithi100 bash[72289]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-node-exporter-a 2024-07-28T19:33:53.744 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:53 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-07-28T19:33:53.744 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:53 smithi100 systemd[1]: Stopped Ceph node-exporter.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:33:53.744 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:53 smithi100 systemd[1]: Started Ceph node-exporter.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:33:54.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:54 smithi138 bash[23952]: audit 2024-07-28T19:33:53.307052+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:54.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:54 smithi138 bash[23952]: audit 2024-07-28T19:33:53.313367+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:54.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:54 smithi138 bash[23952]: cephadm 2024-07-28T19:33:53.313761+0000 mgr.x (mgr.24484) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-28T19:33:54.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:54 smithi138 bash[23952]: cephadm 2024-07-28T19:33:53.341175+0000 mgr.x (mgr.24484) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi100 2024-07-28T19:33:54.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:54 smithi138 bash[23952]: cluster 2024-07-28T19:33:53.932532+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-07-28T19:33:54.631 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[21828]: audit 2024-07-28T19:33:53.307052+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:54.631 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[21828]: audit 2024-07-28T19:33:53.313367+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:54.631 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[21828]: cephadm 2024-07-28T19:33:53.313761+0000 mgr.x (mgr.24484) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-28T19:33:54.631 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[21828]: cephadm 2024-07-28T19:33:53.341175+0000 mgr.x (mgr.24484) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi100 2024-07-28T19:33:54.631 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[21828]: cluster 2024-07-28T19:33:53.932532+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-07-28T19:33:54.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[31715]: audit 2024-07-28T19:33:53.307052+0000 mon.a (mon.0) 1089 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:54.631 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[31715]: audit 2024-07-28T19:33:53.313367+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:33:54.632 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[31715]: cephadm 2024-07-28T19:33:53.313761+0000 mgr.x (mgr.24484) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-07-28T19:33:54.632 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[31715]: cephadm 2024-07-28T19:33:53.341175+0000 mgr.x (mgr.24484) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi100 2024-07-28T19:33:54.632 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:54 smithi100 bash[31715]: cluster 2024-07-28T19:33:53.932532+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-07-28T19:33:54.948 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:33:55.003 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:33:55.019 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:33:55.567 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[21828]: cluster 2024-07-28T19:33:54.344154+0000 mgr.x (mgr.24484) 33 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 12 MiB/s wr, 278 op/s 2024-07-28T19:33:55.567 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[21828]: audit 2024-07-28T19:33:54.382659+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.100:0/691421594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:55.567 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[21828]: cluster 2024-07-28T19:33:54.935527+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-07-28T19:33:55.568 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[31715]: cluster 2024-07-28T19:33:54.344154+0000 mgr.x (mgr.24484) 33 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 12 MiB/s wr, 278 op/s 2024-07-28T19:33:55.568 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[31715]: audit 2024-07-28T19:33:54.382659+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.100:0/691421594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:55.568 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[31715]: cluster 2024-07-28T19:33:54.935527+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-07-28T19:33:55.568 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.555Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-07-28T19:33:55.568 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.555Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-07-28T19:33:55.568 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.556Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.556Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.556Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=arp 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=bcache 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=bonding 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=btrfs 2024-07-28T19:33:55.569 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=conntrack 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=cpu 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=diskstats 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=dmi 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=edac 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=entropy 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=filefd 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=filesystem 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=hwmon 2024-07-28T19:33:55.570 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=infiniband 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=ipvs 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=loadavg 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=mdadm 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=meminfo 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=netclass 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=netdev 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=netstat 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=nfs 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=nfsd 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=nvme 2024-07-28T19:33:55.571 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=os 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=pressure 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=rapl 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=schedstat 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=selinux 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=sockstat 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=softnet 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=stat 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=tapestats 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=textfile 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=time 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-07-28T19:33:55.572 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=uname 2024-07-28T19:33:55.573 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=vmstat 2024-07-28T19:33:55.573 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=xfs 2024-07-28T19:33:55.573 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=node_exporter.go:117 level=info collector=zfs 2024-07-28T19:33:55.573 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-07-28T19:33:55.573 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:33:55 smithi100 bash[72318]: ts=2024-07-28T19:33:55.557Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-07-28T19:33:55.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:55 smithi138 bash[23952]: cluster 2024-07-28T19:33:54.344154+0000 mgr.x (mgr.24484) 33 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 12 MiB/s wr, 278 op/s 2024-07-28T19:33:55.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:55 smithi138 bash[23952]: audit 2024-07-28T19:33:54.382659+0000 mon.c (mon.2) 179 : audit [DBG] from='client.? 172.21.15.100:0/691421594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:55.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:55 smithi138 bash[23952]: cluster 2024-07-28T19:33:54.935527+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-07-28T19:33:56.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:33:56 smithi138 bash[52402]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:56] "GET /metrics HTTP/1.1" 200 38819 "" "Prometheus/2.43.0" 2024-07-28T19:33:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:56 smithi100 bash[31715]: cluster 2024-07-28T19:33:55.939262+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-07-28T19:33:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:56 smithi100 bash[31715]: cluster 2024-07-28T19:33:56.344919+0000 mgr.x (mgr.24484) 34 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 12 MiB/s wr, 278 op/s 2024-07-28T19:33:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:56 smithi100 bash[31715]: audit 2024-07-28T19:33:56.735787+0000 mon.a (mon.0) 1094 : audit [DBG] from='client.? 172.21.15.100:0/2820260674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:57.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:56 smithi100 bash[21828]: cluster 2024-07-28T19:33:55.939262+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-07-28T19:33:57.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:56 smithi100 bash[21828]: cluster 2024-07-28T19:33:56.344919+0000 mgr.x (mgr.24484) 34 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 12 MiB/s wr, 278 op/s 2024-07-28T19:33:57.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:56 smithi100 bash[21828]: audit 2024-07-28T19:33:56.735787+0000 mon.a (mon.0) 1094 : audit [DBG] from='client.? 172.21.15.100:0/2820260674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:57.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:56 smithi138 bash[23952]: cluster 2024-07-28T19:33:55.939262+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-07-28T19:33:57.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:56 smithi138 bash[23952]: cluster 2024-07-28T19:33:56.344919+0000 mgr.x (mgr.24484) 34 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 12 MiB/s wr, 278 op/s 2024-07-28T19:33:57.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:56 smithi138 bash[23952]: audit 2024-07-28T19:33:56.735787+0000 mon.a (mon.0) 1094 : audit [DBG] from='client.? 172.21.15.100:0/2820260674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:33:58.174 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:57 smithi100 bash[21828]: cluster 2024-07-28T19:33:56.954629+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-07-28T19:33:58.174 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:57 smithi100 bash[21828]: cluster 2024-07-28T19:33:57.882456+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-07-28T19:33:58.174 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:57 smithi100 bash[31715]: cluster 2024-07-28T19:33:56.954629+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-07-28T19:33:58.174 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:57 smithi100 bash[31715]: cluster 2024-07-28T19:33:57.882456+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-07-28T19:33:58.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:57 smithi138 bash[23952]: cluster 2024-07-28T19:33:56.954629+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-07-28T19:33:58.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:57 smithi138 bash[23952]: cluster 2024-07-28T19:33:57.882456+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-07-28T19:33:58.442 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:33:58 smithi100 bash[22098]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:33:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:33:58.442 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:33:58 smithi100 systemd[1]: Stopping Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:33:58.442 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:33:58 smithi100 bash[58278]: ts=2024-07-28T19:33:58.409Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-07-28T19:33:58.852 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: after_prepare_snap 2024-07-28T19:33:59.242 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[72600]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-alertmanager-a 2024-07-28T19:33:59.242 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:58 smithi100 bash[21828]: cluster 2024-07-28T19:33:58.345867+0000 mgr.x (mgr.24484) 35 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:59.242 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:58 smithi100 bash[31715]: cluster 2024-07-28T19:33:58.345867+0000 mgr.x (mgr.24484) 35 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:59.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:58 smithi138 bash[23952]: cluster 2024-07-28T19:33:58.345867+0000 mgr.x (mgr.24484) 35 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:33:59.493 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[72686]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-alertmanager-a 2024-07-28T19:33:59.494 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:33:59 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-07-28T19:33:59.494 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:33:59 smithi100 systemd[1]: Stopped Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:33:59.494 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:33:59 smithi100 systemd[1]: Started Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:33:59.980 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 41943040 2024-07-28T19:34:00.057 INFO:tasks.workunit.client.0.smithi100.stdout:write: * 0~1000 2024-07-28T19:34:00.074 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:34:00.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[21828]: cluster 2024-07-28T19:33:58.959429+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[21828]: audit 2024-07-28T19:33:59.114195+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.100:0/558650114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[21828]: audit 2024-07-28T19:33:59.312766+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[21828]: audit 2024-07-28T19:33:59.319043+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[21828]: cephadm 2024-07-28T19:33:59.320914+0000 mgr.x (mgr.24484) 36 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[31715]: cluster 2024-07-28T19:33:58.959429+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[31715]: audit 2024-07-28T19:33:59.114195+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.100:0/558650114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[31715]: audit 2024-07-28T19:33:59.312766+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[31715]: audit 2024-07-28T19:33:59.319043+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:33:59 smithi100 bash[31715]: cephadm 2024-07-28T19:33:59.320914+0000 mgr.x (mgr.24484) 36 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:34:00.273 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:34:00.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:59 smithi138 bash[23952]: cluster 2024-07-28T19:33:58.959429+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-07-28T19:34:00.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:59 smithi138 bash[23952]: audit 2024-07-28T19:33:59.114195+0000 mon.a (mon.0) 1098 : audit [DBG] from='client.? 172.21.15.100:0/558650114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:00.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:59 smithi138 bash[23952]: audit 2024-07-28T19:33:59.312766+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:00.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:59 smithi138 bash[23952]: audit 2024-07-28T19:33:59.319043+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:00.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:33:59 smithi138 bash[23952]: cephadm 2024-07-28T19:33:59.320914+0000 mgr.x (mgr.24484) 36 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:34:00.744 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[72716]: ts=2024-07-28T19:34:00.450Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-07-28T19:34:00.744 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[72716]: ts=2024-07-28T19:34:00.450Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-07-28T19:34:00.744 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[72716]: ts=2024-07-28T19:34:00.452Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.100 port=9094 2024-07-28T19:34:00.744 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[72716]: ts=2024-07-28T19:34:00.453Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-07-28T19:34:00.744 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[72716]: ts=2024-07-28T19:34:00.483Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-28T19:34:00.744 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[72716]: ts=2024-07-28T19:34:00.484Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-07-28T19:34:00.744 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[72716]: ts=2024-07-28T19:34:00.487Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-07-28T19:34:00.744 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[72716]: ts=2024-07-28T19:34:00.487Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-07-28T19:34:01.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:00 smithi138 bash[23952]: cephadm 2024-07-28T19:33:59.615344+0000 mgr.x (mgr.24484) 37 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:34:01.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:00 smithi138 bash[23952]: cluster 2024-07-28T19:33:59.961043+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-07-28T19:34:01.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:00 smithi138 bash[23952]: cluster 2024-07-28T19:34:00.346641+0000 mgr.x (mgr.24484) 38 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 12 KiB/s wr, 207 op/s 2024-07-28T19:34:01.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[31715]: cephadm 2024-07-28T19:33:59.615344+0000 mgr.x (mgr.24484) 37 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:34:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[31715]: cluster 2024-07-28T19:33:59.961043+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-07-28T19:34:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[31715]: cluster 2024-07-28T19:34:00.346641+0000 mgr.x (mgr.24484) 38 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 12 KiB/s wr, 207 op/s 2024-07-28T19:34:01.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[21828]: cephadm 2024-07-28T19:33:59.615344+0000 mgr.x (mgr.24484) 37 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:34:01.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[21828]: cluster 2024-07-28T19:33:59.961043+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-07-28T19:34:01.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:00 smithi100 bash[21828]: cluster 2024-07-28T19:34:00.346641+0000 mgr.x (mgr.24484) 38 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 12 KiB/s wr, 207 op/s 2024-07-28T19:34:02.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:01 smithi100 bash[31715]: cluster 2024-07-28T19:34:00.980594+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-07-28T19:34:02.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:01 smithi100 bash[31715]: audit 2024-07-28T19:34:01.470569+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.100:0/651502215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:01 smithi100 bash[21828]: cluster 2024-07-28T19:34:00.980594+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-07-28T19:34:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:01 smithi100 bash[21828]: audit 2024-07-28T19:34:01.470569+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.100:0/651502215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:01 smithi138 bash[23952]: cluster 2024-07-28T19:34:00.980594+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-07-28T19:34:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:01 smithi138 bash[23952]: audit 2024-07-28T19:34:01.470569+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.100:0/651502215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:02.743 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:02 smithi100 bash[72716]: ts=2024-07-28T19:34:02.453Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000317137s 2024-07-28T19:34:03.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:02 smithi100 bash[31715]: cluster 2024-07-28T19:34:01.994621+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-07-28T19:34:03.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:02 smithi100 bash[31715]: cluster 2024-07-28T19:34:02.347423+0000 mgr.x (mgr.24484) 39 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 12 KiB/s wr, 207 op/s 2024-07-28T19:34:03.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:02 smithi100 bash[31715]: audit 2024-07-28T19:34:02.909919+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:02 smithi100 bash[21828]: cluster 2024-07-28T19:34:01.994621+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-07-28T19:34:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:02 smithi100 bash[21828]: cluster 2024-07-28T19:34:02.347423+0000 mgr.x (mgr.24484) 39 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 12 KiB/s wr, 207 op/s 2024-07-28T19:34:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:02 smithi100 bash[21828]: audit 2024-07-28T19:34:02.909919+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:03.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:02 smithi138 bash[23952]: cluster 2024-07-28T19:34:01.994621+0000 mon.a (mon.0) 1104 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-07-28T19:34:03.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:02 smithi138 bash[23952]: cluster 2024-07-28T19:34:02.347423+0000 mgr.x (mgr.24484) 39 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 12 KiB/s wr, 207 op/s 2024-07-28T19:34:03.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:02 smithi138 bash[23952]: audit 2024-07-28T19:34:02.909919+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:03.580 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (14506 ms) 2024-07-28T19:34:03.580 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-07-28T19:34:03.628 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:03.649 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:34:03.756 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:34:04.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:04 smithi138 bash[23952]: cluster 2024-07-28T19:34:03.001281+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-07-28T19:34:04.128 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:04 smithi138 bash[23952]: audit 2024-07-28T19:34:03.851128+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.100:0/692766493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:04.128 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 systemd[1]: Stopping Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:34:04.128 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.938Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-07-28T19:34:04.128 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.938Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-07-28T19:34:04.128 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.939Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.939Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.939Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.939Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.939Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.939Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.940Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.940Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.940Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-07-28T19:34:04.129 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:03 smithi138 bash[50567]: ts=2024-07-28T19:34:03.940Z caller=main.go:1147 level=info msg="See you next time!" 2024-07-28T19:34:04.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:04 smithi100 bash[21828]: cluster 2024-07-28T19:34:03.001281+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-07-28T19:34:04.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:04 smithi100 bash[21828]: audit 2024-07-28T19:34:03.851128+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.100:0/692766493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:04.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:04 smithi100 bash[31715]: cluster 2024-07-28T19:34:03.001281+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-07-28T19:34:04.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:04 smithi100 bash[31715]: audit 2024-07-28T19:34:03.851128+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.100:0/692766493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:04.793 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:04 smithi138 bash[58745]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-prometheus-a 2024-07-28T19:34:04.793 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:04 smithi138 bash[58808]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-prometheus-a 2024-07-28T19:34:04.793 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:04 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-07-28T19:34:04.793 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:04 smithi138 systemd[1]: Stopped Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:04.793 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:04 smithi138 systemd[1]: Started Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:05.036 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:34:05.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:05 smithi138 bash[23952]: cluster 2024-07-28T19:34:04.001155+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-07-28T19:34:05.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:05 smithi138 bash[23952]: cluster 2024-07-28T19:34:04.348289+0000 mgr.x (mgr.24484) 40 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 6.0 MiB/s wr, 484 op/s 2024-07-28T19:34:05.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:05 smithi138 bash[23952]: audit 2024-07-28T19:34:04.711542+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:05.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:05 smithi138 bash[23952]: audit 2024-07-28T19:34:04.720681+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:05.054 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:04 smithi138 systemd[1]: Stopping Ceph node-exporter.b for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:34:05.065 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:34:05.065 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:34:05.065 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:34:05.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:05 smithi100 bash[21828]: cluster 2024-07-28T19:34:04.001155+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-07-28T19:34:05.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:05 smithi100 bash[21828]: cluster 2024-07-28T19:34:04.348289+0000 mgr.x (mgr.24484) 40 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 6.0 MiB/s wr, 484 op/s 2024-07-28T19:34:05.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:05 smithi100 bash[21828]: audit 2024-07-28T19:34:04.711542+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:05.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:05 smithi100 bash[21828]: audit 2024-07-28T19:34:04.720681+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:05.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:05 smithi100 bash[31715]: cluster 2024-07-28T19:34:04.001155+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-07-28T19:34:05.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:05 smithi100 bash[31715]: cluster 2024-07-28T19:34:04.348289+0000 mgr.x (mgr.24484) 40 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 6.0 MiB/s wr, 484 op/s 2024-07-28T19:34:05.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:05 smithi100 bash[31715]: audit 2024-07-28T19:34:04.711542+0000 mon.a (mon.0) 1108 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:05.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:05 smithi100 bash[31715]: audit 2024-07-28T19:34:04.720681+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:06.054 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58883]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-node-exporter-b 2024-07-28T19:34:06.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[23952]: cephadm 2024-07-28T19:34:04.720342+0000 mgr.x (mgr.24484) 41 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-07-28T19:34:06.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[23952]: cephadm 2024-07-28T19:34:04.720720+0000 mgr.x (mgr.24484) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi138 2024-07-28T19:34:06.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[23952]: cluster 2024-07-28T19:34:05.022405+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-07-28T19:34:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:06 smithi100 bash[21828]: cephadm 2024-07-28T19:34:04.720342+0000 mgr.x (mgr.24484) 41 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-07-28T19:34:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:06 smithi100 bash[21828]: cephadm 2024-07-28T19:34:04.720720+0000 mgr.x (mgr.24484) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi138 2024-07-28T19:34:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:06 smithi100 bash[21828]: cluster 2024-07-28T19:34:05.022405+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-07-28T19:34:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:06 smithi100 bash[31715]: cephadm 2024-07-28T19:34:04.720342+0000 mgr.x (mgr.24484) 41 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-07-28T19:34:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:06 smithi100 bash[31715]: cephadm 2024-07-28T19:34:04.720720+0000 mgr.x (mgr.24484) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi138 2024-07-28T19:34:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:06 smithi100 bash[31715]: cluster 2024-07-28T19:34:05.022405+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-07-28T19:34:06.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:06 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:34:06.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58944]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-node-exporter-b 2024-07-28T19:34:06.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:06 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-07-28T19:34:06.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:06 smithi138 systemd[1]: Stopped Ceph node-exporter.b for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:06.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:06 smithi138 systemd[1]: Started Ceph node-exporter.b for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.526Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.526Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.526Z caller=main.go:570 level=info host_details="(Linux 5.15.0-117-generic #127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024 x86_64 smithi138 (none))" 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.526Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.526Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.528Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.529Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.530Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.530Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.533Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.533Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.442µs 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.533Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-07-28T19:34:06.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.533Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-07-28T19:34:07.008 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.559Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-07-28T19:34:07.008 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.560Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-07-28T19:34:07.008 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.560Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.62µs wal_replay_duration=27.126849ms wbl_replay_duration=154ns total_replay_duration=27.199093ms 2024-07-28T19:34:07.008 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.564Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-07-28T19:34:07.008 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.564Z caller=main.go:1029 level=info msg="TSDB started" 2024-07-28T19:34:07.008 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.564Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-07-28T19:34:07.008 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.591Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=27.110947ms db_storage=1.486µs remote_storage=1.278µs web_handler=380ns query_engine=893ns scrape=302.295µs scrape_sd=140.474µs notify=16.979µs notify_sd=26.213µs rules=26.187404ms tracing=7.197µs 2024-07-28T19:34:07.009 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.591Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-07-28T19:34:07.009 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:06 smithi138 bash[58836]: ts=2024-07-28T19:34:06.591Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-07-28T19:34:07.055 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:34:07.074 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:34:07.074 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:34:07.096 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:34:07.145 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:34:07.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:07 smithi138 bash[23952]: cluster 2024-07-28T19:34:06.032392+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-07-28T19:34:07.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:07 smithi138 bash[23952]: audit 2024-07-28T19:34:06.219769+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.100:0/482970498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:07.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:07 smithi138 bash[23952]: audit 2024-07-28T19:34:06.346531+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:07.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:07 smithi138 bash[23952]: cluster 2024-07-28T19:34:06.348710+0000 mgr.x (mgr.24484) 43 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 6.0 MiB/s wr, 484 op/s 2024-07-28T19:34:07.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:07 smithi138 bash[23952]: cephadm 2024-07-28T19:34:06.355946+0000 mgr.x (mgr.24484) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-07-28T19:34:07.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:07 smithi138 bash[23952]: audit 2024-07-28T19:34:06.356217+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:07.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:07 smithi138 bash[23952]: cephadm 2024-07-28T19:34:06.404128+0000 mgr.x (mgr.24484) 45 : cephadm [INF] Reconfiguring daemon grafana.a on smithi138 2024-07-28T19:34:07.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[21828]: cluster 2024-07-28T19:34:06.032392+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-07-28T19:34:07.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[21828]: audit 2024-07-28T19:34:06.219769+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.100:0/482970498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:07.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[21828]: audit 2024-07-28T19:34:06.346531+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:07.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[21828]: cluster 2024-07-28T19:34:06.348710+0000 mgr.x (mgr.24484) 43 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 6.0 MiB/s wr, 484 op/s 2024-07-28T19:34:07.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[21828]: cephadm 2024-07-28T19:34:06.355946+0000 mgr.x (mgr.24484) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-07-28T19:34:07.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[21828]: audit 2024-07-28T19:34:06.356217+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:07.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[21828]: cephadm 2024-07-28T19:34:06.404128+0000 mgr.x (mgr.24484) 45 : cephadm [INF] Reconfiguring daemon grafana.a on smithi138 2024-07-28T19:34:07.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[31715]: cluster 2024-07-28T19:34:06.032392+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-07-28T19:34:07.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[31715]: audit 2024-07-28T19:34:06.219769+0000 mon.c (mon.2) 181 : audit [DBG] from='client.? 172.21.15.100:0/482970498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:07.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[31715]: audit 2024-07-28T19:34:06.346531+0000 mon.a (mon.0) 1112 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:07.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[31715]: cluster 2024-07-28T19:34:06.348710+0000 mgr.x (mgr.24484) 43 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 6.0 MiB/s wr, 484 op/s 2024-07-28T19:34:07.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[31715]: cephadm 2024-07-28T19:34:06.355946+0000 mgr.x (mgr.24484) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-07-28T19:34:07.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[31715]: audit 2024-07-28T19:34:06.356217+0000 mon.a (mon.0) 1113 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:07.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:07 smithi100 bash[31715]: cephadm 2024-07-28T19:34:06.404128+0000 mgr.x (mgr.24484) 45 : cephadm [INF] Reconfiguring daemon grafana.a on smithi138 2024-07-28T19:34:08.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[23952]: cluster 2024-07-28T19:34:07.040871+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-07-28T19:34:08.289 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[23952]: cluster 2024-07-28T19:34:07.885985+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-07-28T19:34:08.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:08 smithi100 bash[21828]: cluster 2024-07-28T19:34:07.040871+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-07-28T19:34:08.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:08 smithi100 bash[21828]: cluster 2024-07-28T19:34:07.885985+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-07-28T19:34:08.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:08 smithi100 bash[31715]: cluster 2024-07-28T19:34:07.040871+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-07-28T19:34:08.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:08 smithi100 bash[31715]: cluster 2024-07-28T19:34:07.885985+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-07-28T19:34:08.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.350Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-07-28T19:34:08.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.350Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-07-28T19:34:08.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.351Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-07-28T19:34:08.554 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.351Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=node_exporter.go:117 level=info collector=arp 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=node_exporter.go:117 level=info collector=bcache 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=node_exporter.go:117 level=info collector=bonding 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=node_exporter.go:117 level=info collector=btrfs 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=node_exporter.go:117 level=info collector=conntrack 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=node_exporter.go:117 level=info collector=cpu 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.352Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=diskstats 2024-07-28T19:34:08.555 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=dmi 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=edac 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=entropy 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=filefd 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=filesystem 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=hwmon 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=infiniband 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=ipvs 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=loadavg 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=mdadm 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=meminfo 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=netclass 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=netdev 2024-07-28T19:34:08.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=netstat 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=nfs 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=nfsd 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=nvme 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=os 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=pressure 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=rapl 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=schedstat 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=selinux 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=sockstat 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=softnet 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=stat 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=tapestats 2024-07-28T19:34:08.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=textfile 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=time 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=uname 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=vmstat 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=xfs 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=node_exporter.go:117 level=info collector=zfs 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-07-28T19:34:08.558 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:34:08 smithi138 bash[58969]: ts=2024-07-28T19:34:08.353Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-07-28T19:34:08.904 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:08.956 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:34:08.976 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:09.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:09 smithi138 bash[23952]: cluster 2024-07-28T19:34:08.349305+0000 mgr.x (mgr.24484) 46 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 4.0 MiB/s wr, 43 op/s 2024-07-28T19:34:09.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:09 smithi138 bash[23952]: audit 2024-07-28T19:34:08.576804+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.100:0/4044036898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:09.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:09 smithi138 bash[23952]: cluster 2024-07-28T19:34:08.889098+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-07-28T19:34:09.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:09 smithi100 bash[21828]: cluster 2024-07-28T19:34:08.349305+0000 mgr.x (mgr.24484) 46 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 4.0 MiB/s wr, 43 op/s 2024-07-28T19:34:09.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:09 smithi100 bash[21828]: audit 2024-07-28T19:34:08.576804+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.100:0/4044036898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:09.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:09 smithi100 bash[21828]: cluster 2024-07-28T19:34:08.889098+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-07-28T19:34:09.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:09 smithi100 bash[31715]: cluster 2024-07-28T19:34:08.349305+0000 mgr.x (mgr.24484) 46 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 4.0 MiB/s wr, 43 op/s 2024-07-28T19:34:09.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:09 smithi100 bash[31715]: audit 2024-07-28T19:34:08.576804+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.100:0/4044036898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:09.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:09 smithi100 bash[31715]: cluster 2024-07-28T19:34:08.889098+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-07-28T19:34:10.743 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:10 smithi100 bash[72716]: ts=2024-07-28T19:34:10.455Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00195186s 2024-07-28T19:34:11.304 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:11 smithi138 systemd[1]: Stopping Ceph grafana.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:34:11.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:11 smithi138 bash[23952]: cluster 2024-07-28T19:34:10.053074+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-07-28T19:34:11.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:11 smithi138 bash[23952]: cluster 2024-07-28T19:34:10.349725+0000 mgr.x (mgr.24484) 47 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 12 MiB/s wr, 159 op/s 2024-07-28T19:34:11.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:11 smithi138 bash[23952]: audit 2024-07-28T19:34:10.939106+0000 mon.a (mon.0) 1118 : audit [DBG] from='client.? 172.21.15.100:0/3728541727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:11.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:11 smithi100 bash[21828]: cluster 2024-07-28T19:34:10.053074+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-07-28T19:34:11.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:11 smithi100 bash[21828]: cluster 2024-07-28T19:34:10.349725+0000 mgr.x (mgr.24484) 47 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 12 MiB/s wr, 159 op/s 2024-07-28T19:34:11.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:11 smithi100 bash[21828]: audit 2024-07-28T19:34:10.939106+0000 mon.a (mon.0) 1118 : audit [DBG] from='client.? 172.21.15.100:0/3728541727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:11.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:11 smithi100 bash[31715]: cluster 2024-07-28T19:34:10.053074+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-07-28T19:34:11.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:11 smithi100 bash[31715]: cluster 2024-07-28T19:34:10.349725+0000 mgr.x (mgr.24484) 47 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s rd, 12 MiB/s wr, 159 op/s 2024-07-28T19:34:11.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:11 smithi100 bash[31715]: audit 2024-07-28T19:34:10.939106+0000 mon.a (mon.0) 1118 : audit [DBG] from='client.? 172.21.15.100:0/3728541727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:11.967 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:11 smithi138 bash[59258]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-grafana-a 2024-07-28T19:34:11.968 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:11 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:34:11.968 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:11 smithi138 bash[59317]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-grafana-a 2024-07-28T19:34:11.968 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:11 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-07-28T19:34:11.968 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:11 smithi138 systemd[1]: Stopped Ceph grafana.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:11.968 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:11 smithi138 systemd[1]: Started Ceph grafana.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:12.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:12 smithi138 bash[23952]: cluster 2024-07-28T19:34:11.062564+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-07-28T19:34:12.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:12 smithi138 bash[23952]: audit 2024-07-28T19:34:11.901835+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:12.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:12 smithi138 bash[23952]: audit 2024-07-28T19:34:11.910723+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:12.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:12 smithi138 bash[23952]: audit 2024-07-28T19:34:11.915360+0000 mon.a (mon.0) 1122 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:34:12.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:12 smithi138 bash[23952]: audit 2024-07-28T19:34:11.920610+0000 mon.a (mon.0) 1123 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:12.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:12 smithi138 bash[23952]: audit 2024-07-28T19:34:11.925353+0000 mon.a (mon.0) 1124 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:34:12.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:12 smithi138 bash[23952]: audit 2024-07-28T19:34:11.967962+0000 mon.a (mon.0) 1125 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:12.344 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[31715]: cluster 2024-07-28T19:34:11.062564+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-07-28T19:34:12.344 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[31715]: audit 2024-07-28T19:34:11.901835+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:12.344 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[31715]: audit 2024-07-28T19:34:11.910723+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:12.344 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[31715]: audit 2024-07-28T19:34:11.915360+0000 mon.a (mon.0) 1122 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:34:12.344 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[31715]: audit 2024-07-28T19:34:11.920610+0000 mon.a (mon.0) 1123 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[31715]: audit 2024-07-28T19:34:11.925353+0000 mon.a (mon.0) 1124 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[31715]: audit 2024-07-28T19:34:11.967962+0000 mon.a (mon.0) 1125 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[21828]: cluster 2024-07-28T19:34:11.062564+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[21828]: audit 2024-07-28T19:34:11.901835+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[21828]: audit 2024-07-28T19:34:11.910723+0000 mon.a (mon.0) 1121 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[21828]: audit 2024-07-28T19:34:11.915360+0000 mon.a (mon.0) 1122 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[21828]: audit 2024-07-28T19:34:11.920610+0000 mon.a (mon.0) 1123 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[21828]: audit 2024-07-28T19:34:11.925353+0000 mon.a (mon.0) 1124 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:34:12.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:12 smithi100 bash[21828]: audit 2024-07-28T19:34:11.967962+0000 mon.a (mon.0) 1125 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:12.896 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: after_prepare_snap 2024-07-28T19:34:13.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[23952]: audit 2024-07-28T19:34:11.914367+0000 mgr.x (mgr.24484) 48 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:34:13.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[23952]: audit 2024-07-28T19:34:11.919704+0000 mgr.x (mgr.24484) 49 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:13.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[23952]: audit 2024-07-28T19:34:11.924155+0000 mgr.x (mgr.24484) 50 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:34:13.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[23952]: cephadm 2024-07-28T19:34:11.969664+0000 mgr.x (mgr.24484) 51 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-28T19:34:13.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[23952]: cephadm 2024-07-28T19:34:11.970287+0000 mgr.x (mgr.24484) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-28T19:34:13.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[23952]: cluster 2024-07-28T19:34:12.075150+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-07-28T19:34:13.391 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[23952]: cluster 2024-07-28T19:34:12.350305+0000 mgr.x (mgr.24484) 53 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 8.0 MiB/s wr, 116 op/s 2024-07-28T19:34:13.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[21828]: audit 2024-07-28T19:34:11.914367+0000 mgr.x (mgr.24484) 48 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:34:13.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[21828]: audit 2024-07-28T19:34:11.919704+0000 mgr.x (mgr.24484) 49 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:13.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[21828]: audit 2024-07-28T19:34:11.924155+0000 mgr.x (mgr.24484) 50 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:34:13.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[21828]: cephadm 2024-07-28T19:34:11.969664+0000 mgr.x (mgr.24484) 51 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-28T19:34:13.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[21828]: cephadm 2024-07-28T19:34:11.970287+0000 mgr.x (mgr.24484) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-28T19:34:13.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[21828]: cluster 2024-07-28T19:34:12.075150+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-07-28T19:34:13.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[21828]: cluster 2024-07-28T19:34:12.350305+0000 mgr.x (mgr.24484) 53 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 8.0 MiB/s wr, 116 op/s 2024-07-28T19:34:13.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[31715]: audit 2024-07-28T19:34:11.914367+0000 mgr.x (mgr.24484) 48 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-07-28T19:34:13.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[31715]: audit 2024-07-28T19:34:11.919704+0000 mgr.x (mgr.24484) 49 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:13.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[31715]: audit 2024-07-28T19:34:11.924155+0000 mgr.x (mgr.24484) 50 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-07-28T19:34:13.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[31715]: cephadm 2024-07-28T19:34:11.969664+0000 mgr.x (mgr.24484) 51 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-28T19:34:13.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[31715]: cephadm 2024-07-28T19:34:11.970287+0000 mgr.x (mgr.24484) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-07-28T19:34:13.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[31715]: cluster 2024-07-28T19:34:12.075150+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-07-28T19:34:13.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:13 smithi100 bash[31715]: cluster 2024-07-28T19:34:12.350305+0000 mgr.x (mgr.24484) 53 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 15 MiB/s rd, 8.0 MiB/s wr, 116 op/s 2024-07-28T19:34:13.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[58836]: ts=2024-07-28T19:34:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi138\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi138\", job=\"node\", machine=\"x86_64\", nodename=\"smithi138\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi138\", job=\"node\", machine=\"x86_64\", nodename=\"smithi138\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:13.804 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.39568717Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-07-28T19:34:13.804 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.395943117Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.395972576Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.395987075Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.395998704Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.396012178Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.396026306Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.39603901Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.396052476Z level=info msg="Path Home" path=/usr/share/grafana 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.396065673Z level=info msg="Path Data" path=/var/lib/grafana 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.396079638Z level=info msg="Path Logs" path=/var/log/grafana 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.396090713Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.39610137Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-07-28T19:34:13.805 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=settings t=2024-07-28T19:34:13.396111346Z level=info msg="App mode production" 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=sqlstore t=2024-07-28T19:34:13.396193248Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=sqlstore t=2024-07-28T19:34:13.396236926Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=migrator t=2024-07-28T19:34:13.413326448Z level=info msg="Starting DB migrations" 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=migrator t=2024-07-28T19:34:13.419015341Z level=info msg="migrations completed" performed=0 skipped=481 duration=524.513µs 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=sqlstore t=2024-07-28T19:34:13.419443267Z level=info msg="Created default organization" 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=plugin.loader t=2024-07-28T19:34:13.532843504Z level=info msg="Plugin registered" pluginID=input 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=plugin.loader t=2024-07-28T19:34:13.565320556Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=plugin.loader t=2024-07-28T19:34:13.565342536Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=secrets t=2024-07-28T19:34:13.565383578Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=query_data t=2024-07-28T19:34:13.567493523Z level=info msg="Query Service initialization" 2024-07-28T19:34:13.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:13 smithi138 bash[59346]: logger=live.push_http t=2024-07-28T19:34:13.571929963Z level=info msg="Live Push Gateway initialization" 2024-07-28T19:34:14.107 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:34:14.143 INFO:tasks.workunit.client.0.smithi100.stdout:write: * 83886080~1000 2024-07-28T19:34:14.158 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:34:14.409 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:34:14.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:14 smithi100 bash[21828]: cluster 2024-07-28T19:34:13.078462+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-07-28T19:34:14.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:14 smithi100 bash[21828]: audit 2024-07-28T19:34:13.307009+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.100:0/3442957141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:14.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:14 smithi100 bash[31715]: cluster 2024-07-28T19:34:13.078462+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-07-28T19:34:14.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:14 smithi100 bash[31715]: audit 2024-07-28T19:34:13.307009+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.100:0/3442957141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:14.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:14 smithi138 bash[23952]: cluster 2024-07-28T19:34:13.078462+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-07-28T19:34:14.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:14 smithi138 bash[23952]: audit 2024-07-28T19:34:13.307009+0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.100:0/3442957141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:15.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:14 smithi138 bash[59346]: logger=infra.usagestats.collector t=2024-07-28T19:34:14.762779144Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-07-28T19:34:15.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:14 smithi138 bash[59346]: logger=provisioning.datasources t=2024-07-28T19:34:14.878973872Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-07-28T19:34:15.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:14 smithi138 bash[59346]: logger=provisioning.datasources t=2024-07-28T19:34:14.879039054Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-07-28T19:34:15.055 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:14 smithi138 bash[59346]: logger=provisioning.datasources t=2024-07-28T19:34:14.890354927Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-07-28T19:34:15.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[21828]: cluster 2024-07-28T19:34:14.090696+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-07-28T19:34:15.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[21828]: cluster 2024-07-28T19:34:14.350763+0000 mgr.x (mgr.24484) 54 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-07-28T19:34:15.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[21828]: audit 2024-07-28T19:34:14.538341+0000 mon.a (mon.0) 1129 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:15.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[21828]: audit 2024-07-28T19:34:14.540420+0000 mon.a (mon.0) 1130 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:34:15.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[21828]: audit 2024-07-28T19:34:14.542250+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:34:15.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[21828]: audit 2024-07-28T19:34:14.543992+0000 mon.a (mon.0) 1132 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:15.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[31715]: cluster 2024-07-28T19:34:14.090696+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-07-28T19:34:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[31715]: cluster 2024-07-28T19:34:14.350763+0000 mgr.x (mgr.24484) 54 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-07-28T19:34:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[31715]: audit 2024-07-28T19:34:14.538341+0000 mon.a (mon.0) 1129 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[31715]: audit 2024-07-28T19:34:14.540420+0000 mon.a (mon.0) 1130 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:34:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[31715]: audit 2024-07-28T19:34:14.542250+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:34:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:15 smithi100 bash[31715]: audit 2024-07-28T19:34:14.543992+0000 mon.a (mon.0) 1132 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:15.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[23952]: cluster 2024-07-28T19:34:14.090696+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-07-28T19:34:15.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[23952]: cluster 2024-07-28T19:34:14.350763+0000 mgr.x (mgr.24484) 54 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-07-28T19:34:15.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[23952]: audit 2024-07-28T19:34:14.538341+0000 mon.a (mon.0) 1129 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[23952]: audit 2024-07-28T19:34:14.540420+0000 mon.a (mon.0) 1130 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:34:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[23952]: audit 2024-07-28T19:34:14.542250+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:34:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[23952]: audit 2024-07-28T19:34:14.543992+0000 mon.a (mon.0) 1132 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:15.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[59346]: logger=provisioning.alerting t=2024-07-28T19:34:15.103903333Z level=info msg="starting to provision alerting" 2024-07-28T19:34:15.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[59346]: logger=provisioning.alerting t=2024-07-28T19:34:15.103975742Z level=info msg="finished to provision alerting" 2024-07-28T19:34:15.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[59346]: logger=grafanaStorageLogger t=2024-07-28T19:34:15.104659747Z level=info msg="storage starting" 2024-07-28T19:34:15.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[59346]: logger=ngalert.state.manager t=2024-07-28T19:34:15.105266064Z level=info msg="Warming state cache for startup" 2024-07-28T19:34:15.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[59346]: logger=http.server t=2024-07-28T19:34:15.107520359Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-07-28T19:34:15.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[59346]: logger=ngalert.state.manager t=2024-07-28T19:34:15.152780474Z level=info msg="State cache has been initialized" states=0 duration=47.512397ms 2024-07-28T19:34:15.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[59346]: logger=ngalert.multiorg.alertmanager t=2024-07-28T19:34:15.152896608Z level=info msg="starting MultiOrg Alertmanager" 2024-07-28T19:34:15.555 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:34:15 smithi138 bash[59346]: logger=ticker t=2024-07-28T19:34:15.152988758Z level=info msg=starting first_tick=2024-07-28T19:34:20Z 2024-07-28T19:34:16.354 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:16 smithi100 bash[21828]: cephadm 2024-07-28T19:34:14.530443+0000 mgr.x (mgr.24484) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-07-28T19:34:16.354 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:16 smithi100 bash[21828]: cephadm 2024-07-28T19:34:14.543869+0000 mgr.x (mgr.24484) 56 : cephadm [INF] Deploying daemon mgr.y on smithi100 2024-07-28T19:34:16.354 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:16 smithi100 bash[21828]: cluster 2024-07-28T19:34:15.101775+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-07-28T19:34:16.354 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:16 smithi100 bash[21828]: audit 2024-07-28T19:34:15.664224+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.100:0/2835767501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:16.354 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:16 smithi100 bash[31715]: cephadm 2024-07-28T19:34:14.530443+0000 mgr.x (mgr.24484) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-07-28T19:34:16.355 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:16 smithi100 bash[31715]: cephadm 2024-07-28T19:34:14.543869+0000 mgr.x (mgr.24484) 56 : cephadm [INF] Deploying daemon mgr.y on smithi100 2024-07-28T19:34:16.355 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:16 smithi100 bash[31715]: cluster 2024-07-28T19:34:15.101775+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-07-28T19:34:16.355 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:16 smithi100 bash[31715]: audit 2024-07-28T19:34:15.664224+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.100:0/2835767501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:16 smithi138 bash[23952]: cephadm 2024-07-28T19:34:14.530443+0000 mgr.x (mgr.24484) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-07-28T19:34:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:16 smithi138 bash[23952]: cephadm 2024-07-28T19:34:14.543869+0000 mgr.x (mgr.24484) 56 : cephadm [INF] Deploying daemon mgr.y on smithi100 2024-07-28T19:34:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:16 smithi138 bash[23952]: cluster 2024-07-28T19:34:15.101775+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-07-28T19:34:16.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:16 smithi138 bash[23952]: audit 2024-07-28T19:34:15.664224+0000 mon.c (mon.2) 184 : audit [DBG] from='client.? 172.21.15.100:0/2835767501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:17.191 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.191 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.192 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.192 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.192 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.192 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:17 smithi100 bash[21828]: cluster 2024-07-28T19:34:16.099226+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-07-28T19:34:17.193 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:17 smithi100 bash[21828]: cluster 2024-07-28T19:34:16.351537+0000 mgr.x (mgr.24484) 57 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-07-28T19:34:17.193 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.193 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.193 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:17 smithi100 bash[31715]: cluster 2024-07-28T19:34:16.099226+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-07-28T19:34:17.193 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:17 smithi100 bash[31715]: cluster 2024-07-28T19:34:16.351537+0000 mgr.x (mgr.24484) 57 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-07-28T19:34:17.194 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.194 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:17.493 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:17 smithi100 systemd[1]: Stopping Ceph mgr.y for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:34:17.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:17 smithi138 bash[23952]: cluster 2024-07-28T19:34:16.099226+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-07-28T19:34:17.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:17 smithi138 bash[23952]: cluster 2024-07-28T19:34:16.351537+0000 mgr.x (mgr.24484) 57 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 278 op/s 2024-07-28T19:34:17.791 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (14211 ms) 2024-07-28T19:34:17.792 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.Clone 2024-07-28T19:34:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:17.885 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~8388609 2024-07-28T19:34:17.998 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 16777218~8388609 2024-07-28T19:34:18.056 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap 2024-07-28T19:34:18.239 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:18 smithi100 bash[21828]: cluster 2024-07-28T19:34:17.121773+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-07-28T19:34:18.240 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:18 smithi100 bash[21828]: audit 2024-07-28T19:34:17.910956+0000 mon.a (mon.0) 1136 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:18.240 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:18 smithi100 bash[21828]: audit 2024-07-28T19:34:18.075941+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.100:0/1080861950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:18.240 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:18 smithi100 bash[31715]: cluster 2024-07-28T19:34:17.121773+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-07-28T19:34:18.240 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:18 smithi100 bash[31715]: audit 2024-07-28T19:34:17.910956+0000 mon.a (mon.0) 1136 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:18.240 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:18 smithi100 bash[31715]: audit 2024-07-28T19:34:18.075941+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.100:0/1080861950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:18.240 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:17 smithi100 bash[73167]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mgr-y 2024-07-28T19:34:18.240 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:34:18.240 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:18 smithi100 bash[73248]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mgr-y 2024-07-28T19:34:18.241 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-07-28T19:34:18.241 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: Stopped Ceph mgr.y for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:18.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.494 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.494 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: Started Ceph mgr.y for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:18.495 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.495 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.495 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.495 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:34:18 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:34:18.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:18 smithi138 bash[23952]: cluster 2024-07-28T19:34:17.121773+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-07-28T19:34:18.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:18 smithi138 bash[23952]: audit 2024-07-28T19:34:17.910956+0000 mon.a (mon.0) 1136 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:18.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:18 smithi138 bash[23952]: audit 2024-07-28T19:34:18.075941+0000 mon.c (mon.2) 185 : audit [DBG] from='client.? 172.21.15.100:0/1080861950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:19.743 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[73305]: debug 2024-07-28T19:34:19.643+0000 7f5394e53640 1 -- 172.21.15.100:0/1079128271 <== mon.0 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e30c078d00 con 0x55e30c06b800 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[21828]: cluster 2024-07-28T19:34:18.353304+0000 mgr.x (mgr.24484) 58 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 753 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 11 MiB/s wr, 355 op/s 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[21828]: audit 2024-07-28T19:34:18.402471+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[21828]: audit 2024-07-28T19:34:18.409715+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[21828]: audit 2024-07-28T19:34:18.411392+0000 mon.a (mon.0) 1139 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[21828]: cluster 2024-07-28T19:34:18.896321+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[31715]: cluster 2024-07-28T19:34:18.353304+0000 mgr.x (mgr.24484) 58 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 753 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 11 MiB/s wr, 355 op/s 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[31715]: audit 2024-07-28T19:34:18.402471+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[31715]: audit 2024-07-28T19:34:18.409715+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[31715]: audit 2024-07-28T19:34:18.411392+0000 mon.a (mon.0) 1139 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:19.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[31715]: cluster 2024-07-28T19:34:18.896321+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-07-28T19:34:19.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:19 smithi138 bash[23952]: cluster 2024-07-28T19:34:18.353304+0000 mgr.x (mgr.24484) 58 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 753 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 11 MiB/s wr, 355 op/s 2024-07-28T19:34:19.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:19 smithi138 bash[23952]: audit 2024-07-28T19:34:18.402471+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:19.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:19 smithi138 bash[23952]: audit 2024-07-28T19:34:18.409715+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:19.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:19 smithi138 bash[23952]: audit 2024-07-28T19:34:18.411392+0000 mon.a (mon.0) 1139 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:19.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:19 smithi138 bash[23952]: cluster 2024-07-28T19:34:18.896321+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-07-28T19:34:19.917 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap 2024-07-28T19:34:19.950 INFO:tasks.workunit.client.0.smithi100.stdout:clone image103 -> image106 2024-07-28T19:34:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:34:20.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:19 smithi100 bash[73305]: debug 2024-07-28T19:34:19.759+0000 7f53976bc040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:34:20.244 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:34:20.249 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap 2024-07-28T19:34:21.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:20 smithi138 bash[52402]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:34:20] "GET /metrics HTTP/1.1" 200 38815 "" "Prometheus/2.43.0" 2024-07-28T19:34:21.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:20 smithi138 bash[23952]: cluster 2024-07-28T19:34:19.898103+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-07-28T19:34:21.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:20 smithi138 bash[23952]: cluster 2024-07-28T19:34:20.354107+0000 mgr.x (mgr.24484) 59 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 405 op/s 2024-07-28T19:34:21.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:20 smithi138 bash[23952]: audit 2024-07-28T19:34:20.436161+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.100:0/2342124359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:21.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:21 smithi100 bash[73305]: debug 2024-07-28T19:34:21.043+0000 7f53976bc040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:34:21.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:21 smithi100 bash[73305]: debug 2024-07-28T19:34:21.111+0000 7f53976bc040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:34:21.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:20 smithi100 bash[21828]: cluster 2024-07-28T19:34:19.898103+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-07-28T19:34:21.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:20 smithi100 bash[21828]: cluster 2024-07-28T19:34:20.354107+0000 mgr.x (mgr.24484) 59 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 405 op/s 2024-07-28T19:34:21.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:20 smithi100 bash[21828]: audit 2024-07-28T19:34:20.436161+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.100:0/2342124359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:21.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:20 smithi100 bash[31715]: cluster 2024-07-28T19:34:19.898103+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-07-28T19:34:21.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:20 smithi100 bash[31715]: cluster 2024-07-28T19:34:20.354107+0000 mgr.x (mgr.24484) 59 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 405 op/s 2024-07-28T19:34:21.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:20 smithi100 bash[31715]: audit 2024-07-28T19:34:20.436161+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.100:0/2342124359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:21.743 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:21 smithi100 bash[73305]: debug 2024-07-28T19:34:21.267+0000 7f53976bc040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:34:21.938 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap 2024-07-28T19:34:21.964 INFO:tasks.workunit.client.0.smithi100.stdout:clone image106 -> image108 2024-07-28T19:34:22.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:21 smithi100 bash[21828]: cluster 2024-07-28T19:34:20.912683+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-07-28T19:34:22.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:21 smithi100 bash[73305]: debug 2024-07-28T19:34:21.943+0000 7f53976bc040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:34:22.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[73305]: debug 2024-07-28T19:34:22.027+0000 7f53976bc040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:34:22.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[73305]: debug 2024-07-28T19:34:22.103+0000 7f53976bc040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:34:22.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:21 smithi100 bash[31715]: cluster 2024-07-28T19:34:20.912683+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-07-28T19:34:22.247 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 1000~1000 2024-07-28T19:34:22.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:21 smithi138 bash[23952]: cluster 2024-07-28T19:34:20.912683+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-07-28T19:34:22.348 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 2010~1000 2024-07-28T19:34:22.354 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:22.743 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[73305]: debug 2024-07-28T19:34:22.315+0000 7f53976bc040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:34:22.743 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[73305]: debug 2024-07-28T19:34:22.403+0000 7f53976bc040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:34:22.744 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[73305]: debug 2024-07-28T19:34:22.487+0000 7f53976bc040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:34:22.966 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:34:23.101 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[73305]: debug 2024-07-28T19:34:22.803+0000 7f53976bc040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:34:23.102 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[73305]: debug 2024-07-28T19:34:22.879+0000 7f53976bc040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:34:23.102 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:23 smithi100 bash[73305]: debug 2024-07-28T19:34:23.103+0000 7f53976bc040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:34:23.102 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[21828]: cluster 2024-07-28T19:34:21.920624+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-07-28T19:34:23.103 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[21828]: cluster 2024-07-28T19:34:22.354906+0000 mgr.x (mgr.24484) 60 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 53 op/s 2024-07-28T19:34:23.103 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[21828]: audit 2024-07-28T19:34:22.809507+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.100:0/2586498974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:23.103 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[21828]: cluster 2024-07-28T19:34:22.893622+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-07-28T19:34:23.103 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[31715]: cluster 2024-07-28T19:34:21.920624+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-07-28T19:34:23.103 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[31715]: cluster 2024-07-28T19:34:22.354906+0000 mgr.x (mgr.24484) 60 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 53 op/s 2024-07-28T19:34:23.104 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[31715]: audit 2024-07-28T19:34:22.809507+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.100:0/2586498974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:23.104 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:22 smithi100 bash[31715]: cluster 2024-07-28T19:34:22.893622+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-07-28T19:34:23.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:22 smithi138 bash[23952]: cluster 2024-07-28T19:34:21.920624+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-07-28T19:34:23.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:22 smithi138 bash[23952]: cluster 2024-07-28T19:34:22.354906+0000 mgr.x (mgr.24484) 60 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 761 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 53 op/s 2024-07-28T19:34:23.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:22 smithi138 bash[23952]: audit 2024-07-28T19:34:22.809507+0000 mon.c (mon.2) 187 : audit [DBG] from='client.? 172.21.15.100:0/2586498974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:23.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:22 smithi138 bash[23952]: cluster 2024-07-28T19:34:22.893622+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-07-28T19:34:23.350 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:34:23.493 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:23 smithi100 bash[73305]: debug 2024-07-28T19:34:23.195+0000 7f53976bc040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:34:23.557 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.Clone (5766 ms) 2024-07-28T19:34:23.557 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneParent 2024-07-28T19:34:23.591 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:23.602 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap 2024-07-28T19:34:23.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:23 smithi138 bash[58836]: ts=2024-07-28T19:34:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:23.888 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:23 smithi100 bash[73305]: debug 2024-07-28T19:34:23.499+0000 7f53976bc040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-28T19:34:23.888 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:23 smithi100 bash[73305]: debug 2024-07-28T19:34:23.707+0000 7f53976bc040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:34:24.172 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:23 smithi100 bash[73305]: debug 2024-07-28T19:34:23.891+0000 7f53976bc040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:34:24.172 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:23 smithi100 bash[73305]: debug 2024-07-28T19:34:23.983+0000 7f53976bc040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:34:24.172 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[73305]: debug 2024-07-28T19:34:24.175+0000 7f53976bc040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:34:24.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[58836]: ts=2024-07-28T19:34:24.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:24.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[58836]: ts=2024-07-28T19:34:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:24.950 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap 2024-07-28T19:34:24.967 INFO:tasks.workunit.client.0.smithi100.stdout:clone image109 -> image112 2024-07-28T19:34:25.129 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:25.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[21828]: cluster 2024-07-28T19:34:23.931866+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-07-28T19:34:25.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[21828]: audit 2024-07-28T19:34:23.980737+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[21828]: audit 2024-07-28T19:34:23.987207+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[21828]: audit 2024-07-28T19:34:24.229607+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[21828]: audit 2024-07-28T19:34:24.236728+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[21828]: cluster 2024-07-28T19:34:24.355618+0000 mgr.x (mgr.24484) 61 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 675 op/s 2024-07-28T19:34:25.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[21828]: audit 2024-07-28T19:34:24.716468+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[21828]: audit 2024-07-28T19:34:24.722926+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[73305]: debug 2024-07-28T19:34:25.083+0000 7f53976bc040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:34:25.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[73305]: debug 2024-07-28T19:34:25.175+0000 7f53976bc040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:34:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[31715]: cluster 2024-07-28T19:34:23.931866+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-07-28T19:34:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[31715]: audit 2024-07-28T19:34:23.980737+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[31715]: audit 2024-07-28T19:34:23.987207+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[31715]: audit 2024-07-28T19:34:24.229607+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[31715]: audit 2024-07-28T19:34:24.236728+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[31715]: cluster 2024-07-28T19:34:24.355618+0000 mgr.x (mgr.24484) 61 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 675 op/s 2024-07-28T19:34:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[31715]: audit 2024-07-28T19:34:24.716468+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:24 smithi100 bash[31715]: audit 2024-07-28T19:34:24.722926+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[23952]: cluster 2024-07-28T19:34:23.931866+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-07-28T19:34:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[23952]: audit 2024-07-28T19:34:23.980737+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[23952]: audit 2024-07-28T19:34:23.987207+0000 mon.a (mon.0) 1147 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[23952]: audit 2024-07-28T19:34:24.229607+0000 mon.a (mon.0) 1148 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[23952]: audit 2024-07-28T19:34:24.236728+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[23952]: cluster 2024-07-28T19:34:24.355618+0000 mgr.x (mgr.24484) 61 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 675 op/s 2024-07-28T19:34:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[23952]: audit 2024-07-28T19:34:24.716468+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:24 smithi138 bash[23952]: audit 2024-07-28T19:34:24.722926+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:25.431 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:34:25.704 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneParent (2147 ms) 2024-07-28T19:34:25.705 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-07-28T19:34:25.743 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[73305]: debug 2024-07-28T19:34:25.443+0000 7f53976bc040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:34:25.743 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[73305]: debug 2024-07-28T19:34:25.531+0000 7f53976bc040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:34:25.771 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:25.796 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~10 2024-07-28T19:34:25.814 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap 2024-07-28T19:34:26.185 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[21828]: cluster 2024-07-28T19:34:24.931827+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-07-28T19:34:26.185 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[21828]: audit 2024-07-28T19:34:25.182019+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.100:0/1547154751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:26.185 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[31715]: cluster 2024-07-28T19:34:24.931827+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-07-28T19:34:26.186 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[31715]: audit 2024-07-28T19:34:25.182019+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.100:0/1547154751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:26.186 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:25 smithi100 bash[73305]: debug 2024-07-28T19:34:25.975+0000 7f53976bc040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:34:26.186 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[73305]: debug 2024-07-28T19:34:26.059+0000 7f53976bc040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:34:26.186 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[73305]: [28/Jul/2024:19:34:26] ENGINE Bus STARTING 2024-07-28T19:34:26.186 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[73305]: CherryPy Checker: 2024-07-28T19:34:26.186 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[73305]: The Application mounted at '' has an empty config. 2024-07-28T19:34:26.186 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[73305]: [28/Jul/2024:19:34:26] ENGINE Serving on http://:::9283 2024-07-28T19:34:26.186 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[73305]: [28/Jul/2024:19:34:26] ENGINE Bus STARTED 2024-07-28T19:34:26.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:25 smithi138 bash[23952]: cluster 2024-07-28T19:34:24.931827+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-07-28T19:34:26.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:25 smithi138 bash[23952]: audit 2024-07-28T19:34:25.182019+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.100:0/1547154751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:27.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[21828]: cluster 2024-07-28T19:34:26.082789+0000 mon.a (mon.0) 1153 : cluster [DBG] Standby manager daemon y restarted 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[21828]: cluster 2024-07-28T19:34:26.083025+0000 mon.a (mon.0) 1154 : cluster [DBG] Standby manager daemon y started 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[21828]: audit 2024-07-28T19:34:26.088617+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[21828]: audit 2024-07-28T19:34:26.089648+0000 mon.a (mon.0) 1156 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[21828]: audit 2024-07-28T19:34:26.092534+0000 mon.a (mon.0) 1157 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[21828]: audit 2024-07-28T19:34:26.093016+0000 mon.a (mon.0) 1158 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[21828]: cluster 2024-07-28T19:34:26.246137+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[21828]: cluster 2024-07-28T19:34:26.356434+0000 mgr.x (mgr.24484) 62 : cluster [DBG] pgmap v83: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 675 op/s 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[31715]: cluster 2024-07-28T19:34:26.082789+0000 mon.a (mon.0) 1153 : cluster [DBG] Standby manager daemon y restarted 2024-07-28T19:34:27.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[31715]: cluster 2024-07-28T19:34:26.083025+0000 mon.a (mon.0) 1154 : cluster [DBG] Standby manager daemon y started 2024-07-28T19:34:27.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[31715]: audit 2024-07-28T19:34:26.088617+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-28T19:34:27.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[31715]: audit 2024-07-28T19:34:26.089648+0000 mon.a (mon.0) 1156 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:34:27.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[31715]: audit 2024-07-28T19:34:26.092534+0000 mon.a (mon.0) 1157 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-28T19:34:27.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[31715]: audit 2024-07-28T19:34:26.093016+0000 mon.a (mon.0) 1158 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:34:27.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[31715]: cluster 2024-07-28T19:34:26.246137+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-07-28T19:34:27.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:26 smithi100 bash[31715]: cluster 2024-07-28T19:34:26.356434+0000 mgr.x (mgr.24484) 62 : cluster [DBG] pgmap v83: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 675 op/s 2024-07-28T19:34:27.272 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap 2024-07-28T19:34:27.291 INFO:tasks.workunit.client.0.smithi100.stdout:clone image113 -> image116 2024-07-28T19:34:27.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:26 smithi138 bash[23952]: cluster 2024-07-28T19:34:26.082789+0000 mon.a (mon.0) 1153 : cluster [DBG] Standby manager daemon y restarted 2024-07-28T19:34:27.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:26 smithi138 bash[23952]: cluster 2024-07-28T19:34:26.083025+0000 mon.a (mon.0) 1154 : cluster [DBG] Standby manager daemon y started 2024-07-28T19:34:27.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:26 smithi138 bash[23952]: audit 2024-07-28T19:34:26.088617+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-07-28T19:34:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:26 smithi138 bash[23952]: audit 2024-07-28T19:34:26.089648+0000 mon.a (mon.0) 1156 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:34:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:26 smithi138 bash[23952]: audit 2024-07-28T19:34:26.092534+0000 mon.a (mon.0) 1157 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-07-28T19:34:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:26 smithi138 bash[23952]: audit 2024-07-28T19:34:26.093016+0000 mon.a (mon.0) 1158 : audit [DBG] from='mgr.? 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:34:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:26 smithi138 bash[23952]: cluster 2024-07-28T19:34:26.246137+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-07-28T19:34:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:26 smithi138 bash[23952]: cluster 2024-07-28T19:34:26.356434+0000 mgr.x (mgr.24484) 62 : cluster [DBG] pgmap v83: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 14 MiB/s rd, 5.0 MiB/s wr, 675 op/s 2024-07-28T19:34:27.457 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:27.759 INFO:tasks.workunit.client.0.smithi100.stdout:write: Y 0~1 2024-07-28T19:34:27.818 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:34:28.019 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:34:28.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:27 smithi100 bash[21828]: cluster 2024-07-28T19:34:26.960534+0000 mon.a (mon.0) 1160 : cluster [DBG] mgrmap e27: x(active, since 54s), standbys: y 2024-07-28T19:34:28.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:27 smithi100 bash[21828]: cluster 2024-07-28T19:34:27.257849+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-07-28T19:34:28.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:27 smithi100 bash[21828]: audit 2024-07-28T19:34:27.582308+0000 mon.a (mon.0) 1162 : audit [DBG] from='client.? 172.21.15.100:0/2054636705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:28.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:27 smithi100 bash[31715]: cluster 2024-07-28T19:34:26.960534+0000 mon.a (mon.0) 1160 : cluster [DBG] mgrmap e27: x(active, since 54s), standbys: y 2024-07-28T19:34:28.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:27 smithi100 bash[31715]: cluster 2024-07-28T19:34:27.257849+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-07-28T19:34:28.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:27 smithi100 bash[31715]: audit 2024-07-28T19:34:27.582308+0000 mon.a (mon.0) 1162 : audit [DBG] from='client.? 172.21.15.100:0/2054636705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:28.288 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (2584 ms) 2024-07-28T19:34:28.288 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-07-28T19:34:28.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:27 smithi138 bash[23952]: cluster 2024-07-28T19:34:26.960534+0000 mon.a (mon.0) 1160 : cluster [DBG] mgrmap e27: x(active, since 54s), standbys: y 2024-07-28T19:34:28.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:27 smithi138 bash[23952]: cluster 2024-07-28T19:34:27.257849+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-07-28T19:34:28.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:27 smithi138 bash[23952]: audit 2024-07-28T19:34:27.582308+0000 mon.a (mon.0) 1162 : audit [DBG] from='client.? 172.21.15.100:0/2054636705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:28.370 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:28.399 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 41943040~10 2024-07-28T19:34:28.421 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:34:28.957 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:28 smithi138 bash[58836]: ts=2024-07-28T19:34:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:29.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:28 smithi100 bash[21828]: cluster 2024-07-28T19:34:28.357943+0000 mgr.x (mgr.24484) 63 : cluster [DBG] pgmap v85: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 8.7 MiB/s rd, 956 KiB/s wr, 782 op/s 2024-07-28T19:34:29.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:28 smithi100 bash[31715]: cluster 2024-07-28T19:34:28.357943+0000 mgr.x (mgr.24484) 63 : cluster [DBG] pgmap v85: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 8.7 MiB/s rd, 956 KiB/s wr, 782 op/s 2024-07-28T19:34:29.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:28 smithi138 bash[23952]: cluster 2024-07-28T19:34:28.357943+0000 mgr.x (mgr.24484) 63 : cluster [DBG] pgmap v85: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 8.7 MiB/s rd, 956 KiB/s wr, 782 op/s 2024-07-28T19:34:29.990 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 41943041~10 2024-07-28T19:34:30.001 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:30.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:29 smithi100 bash[21828]: cluster 2024-07-28T19:34:28.974928+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-07-28T19:34:30.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:29 smithi100 bash[21828]: audit 2024-07-28T19:34:29.937751+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.100:0/3711150951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:30.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:29 smithi100 bash[31715]: cluster 2024-07-28T19:34:28.974928+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-07-28T19:34:30.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:29 smithi100 bash[31715]: audit 2024-07-28T19:34:29.937751+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.100:0/3711150951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:30.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:29 smithi138 bash[23952]: cluster 2024-07-28T19:34:28.974928+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-07-28T19:34:30.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:29 smithi138 bash[23952]: audit 2024-07-28T19:34:29.937751+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.100:0/3711150951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:30.978 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:30 smithi138 bash[52402]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:34:30] "GET /metrics HTTP/1.1" 200 38815 "" "Prometheus/2.43.0" 2024-07-28T19:34:31.212 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 41943042~10 2024-07-28T19:34:31.236 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:34:31.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:30 smithi100 bash[31715]: cluster 2024-07-28T19:34:29.973921+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-07-28T19:34:31.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:30 smithi100 bash[31715]: cluster 2024-07-28T19:34:30.358745+0000 mgr.x (mgr.24484) 64 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.0 MiB/s wr, 841 op/s 2024-07-28T19:34:31.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:30 smithi100 bash[21828]: cluster 2024-07-28T19:34:29.973921+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-07-28T19:34:31.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:30 smithi100 bash[21828]: cluster 2024-07-28T19:34:30.358745+0000 mgr.x (mgr.24484) 64 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.0 MiB/s wr, 841 op/s 2024-07-28T19:34:31.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:30 smithi138 bash[23952]: cluster 2024-07-28T19:34:29.973921+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-07-28T19:34:31.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:30 smithi138 bash[23952]: cluster 2024-07-28T19:34:30.358745+0000 mgr.x (mgr.24484) 64 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.0 MiB/s wr, 841 op/s 2024-07-28T19:34:32.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:31 smithi100 bash[31715]: cluster 2024-07-28T19:34:30.995525+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-07-28T19:34:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:31 smithi100 bash[21828]: cluster 2024-07-28T19:34:30.995525+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-07-28T19:34:32.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:31 smithi138 bash[23952]: cluster 2024-07-28T19:34:30.995525+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-07-28T19:34:32.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:32 smithi138 bash[58836]: ts=2024-07-28T19:34:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:33.022 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 41943040 2024-07-28T19:34:33.068 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:34:33.156 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:34:33.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:33 smithi100 bash[21828]: cluster 2024-07-28T19:34:32.011502+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-07-28T19:34:33.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:33 smithi100 bash[21828]: audit 2024-07-28T19:34:32.302503+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.100:0/589693575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:33.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:33 smithi100 bash[21828]: cluster 2024-07-28T19:34:32.359495+0000 mgr.x (mgr.24484) 65 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:33.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:33 smithi100 bash[21828]: audit 2024-07-28T19:34:32.937874+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:33 smithi100 bash[31715]: cluster 2024-07-28T19:34:32.011502+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-07-28T19:34:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:33 smithi100 bash[31715]: audit 2024-07-28T19:34:32.302503+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.100:0/589693575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:33 smithi100 bash[31715]: cluster 2024-07-28T19:34:32.359495+0000 mgr.x (mgr.24484) 65 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:33 smithi100 bash[31715]: audit 2024-07-28T19:34:32.937874+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:33.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:33 smithi138 bash[23952]: cluster 2024-07-28T19:34:32.011502+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-07-28T19:34:33.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:33 smithi138 bash[23952]: audit 2024-07-28T19:34:32.302503+0000 mon.a (mon.0) 1167 : audit [DBG] from='client.? 172.21.15.100:0/589693575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:33.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:33 smithi138 bash[23952]: cluster 2024-07-28T19:34:32.359495+0000 mgr.x (mgr.24484) 65 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:33.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:33 smithi138 bash[23952]: audit 2024-07-28T19:34:32.937874+0000 mon.a (mon.0) 1168 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:33.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:33 smithi138 bash[58836]: ts=2024-07-28T19:34:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:34.214 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (5926 ms) 2024-07-28T19:34:34.215 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-07-28T19:34:34.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:34 smithi100 bash[21828]: cluster 2024-07-28T19:34:33.017066+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-07-28T19:34:34.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:34 smithi100 bash[31715]: cluster 2024-07-28T19:34:33.017066+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-07-28T19:34:34.298 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:34.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:34 smithi138 bash[23952]: cluster 2024-07-28T19:34:33.017066+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-07-28T19:34:34.324 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 41943040~10 2024-07-28T19:34:34.348 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:34:34.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:34 smithi138 bash[58836]: ts=2024-07-28T19:34:34.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:34.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:34 smithi138 bash[58836]: ts=2024-07-28T19:34:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:35.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:34 smithi100 bash[73305]: [28/Jul/2024:19:34:34] ENGINE Bus STOPPING 2024-07-28T19:34:35.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: [28/Jul/2024:19:34:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:34:35.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: [28/Jul/2024:19:34:35] ENGINE Bus STOPPED 2024-07-28T19:34:35.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: cluster 2024-07-28T19:34:34.025040+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-07-28T19:34:35.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: cluster 2024-07-28T19:34:34.360382+0000 mgr.x (mgr.24484) 66 : cluster [DBG] pgmap v94: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 231 KiB/s rd, 22 KiB/s wr, 309 op/s 2024-07-28T19:34:35.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.661845+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.100:0/552531874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.768761+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.780009+0000 mon.a (mon.0) 1172 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.782054+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.784203+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.794179+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.945294+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.953992+0000 mon.a (mon.0) 1177 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: cluster 2024-07-28T19:34:34.954918+0000 mon.a (mon.0) 1178 : cluster [INF] Activating manager daemon y 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: cluster 2024-07-28T19:34:34.967437+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.968741+0000 mon.a (mon.0) 1180 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: cluster 2024-07-28T19:34:34.968868+0000 mon.a (mon.0) 1181 : cluster [DBG] mgrmap e28: y(active, starting, since 0.0139439s) 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.973880+0000 mon.a (mon.0) 1182 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:34:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.974357+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.974749+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.975261+0000 mon.a (mon.0) 1185 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.976014+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.977352+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.978363+0000 mon.a (mon.0) 1188 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.979348+0000 mon.a (mon.0) 1189 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.980453+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.981233+0000 mon.a (mon.0) 1191 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.981758+0000 mon.a (mon.0) 1192 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.982766+0000 mon.a (mon.0) 1193 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:34:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.983201+0000 mon.a (mon.0) 1194 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.983654+0000 mon.a (mon.0) 1195 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.984207+0000 mon.a (mon.0) 1196 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.984861+0000 mon.a (mon.0) 1197 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[21828]: audit 2024-07-28T19:34:34.985953+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: cluster 2024-07-28T19:34:34.025040+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: cluster 2024-07-28T19:34:34.360382+0000 mgr.x (mgr.24484) 66 : cluster [DBG] pgmap v94: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 231 KiB/s rd, 22 KiB/s wr, 309 op/s 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.661845+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.100:0/552531874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.768761+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.780009+0000 mon.a (mon.0) 1172 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.782054+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.784203+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.794179+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.945294+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.953992+0000 mon.a (mon.0) 1177 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: cluster 2024-07-28T19:34:34.954918+0000 mon.a (mon.0) 1178 : cluster [INF] Activating manager daemon y 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: cluster 2024-07-28T19:34:34.967437+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.968741+0000 mon.a (mon.0) 1180 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: cluster 2024-07-28T19:34:34.968868+0000 mon.a (mon.0) 1181 : cluster [DBG] mgrmap e28: y(active, starting, since 0.0139439s) 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.973880+0000 mon.a (mon.0) 1182 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.974357+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:34:35.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.974749+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.975261+0000 mon.a (mon.0) 1185 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.976014+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.977352+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.978363+0000 mon.a (mon.0) 1188 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.979348+0000 mon.a (mon.0) 1189 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.980453+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.981233+0000 mon.a (mon.0) 1191 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.981758+0000 mon.a (mon.0) 1192 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.982766+0000 mon.a (mon.0) 1193 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.983201+0000 mon.a (mon.0) 1194 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.983654+0000 mon.a (mon.0) 1195 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.984207+0000 mon.a (mon.0) 1196 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.984861+0000 mon.a (mon.0) 1197 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:34:35.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[31715]: audit 2024-07-28T19:34:34.985953+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:34:35.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[52402]: ignoring --setuser ceph since I am not root 2024-07-28T19:34:35.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[52402]: ignoring --setgroup ceph since I am not root 2024-07-28T19:34:35.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[52402]: debug 2024-07-28T19:34:35.134+0000 7ff7cf3c2040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:34:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: cluster 2024-07-28T19:34:34.025040+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-07-28T19:34:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: cluster 2024-07-28T19:34:34.360382+0000 mgr.x (mgr.24484) 66 : cluster [DBG] pgmap v94: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 231 KiB/s rd, 22 KiB/s wr, 309 op/s 2024-07-28T19:34:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.661845+0000 mon.c (mon.2) 190 : audit [DBG] from='client.? 172.21.15.100:0/552531874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.768761+0000 mon.a (mon.0) 1171 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.780009+0000 mon.a (mon.0) 1172 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.782054+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.784203+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.794179+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.945294+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.953992+0000 mon.a (mon.0) 1177 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: cluster 2024-07-28T19:34:34.954918+0000 mon.a (mon.0) 1178 : cluster [INF] Activating manager daemon y 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: cluster 2024-07-28T19:34:34.967437+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.968741+0000 mon.a (mon.0) 1180 : audit [INF] from='mgr.24484 172.21.15.138:0/3574639355' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: cluster 2024-07-28T19:34:34.968868+0000 mon.a (mon.0) 1181 : cluster [DBG] mgrmap e28: y(active, starting, since 0.0139439s) 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.973880+0000 mon.a (mon.0) 1182 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.974357+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.974749+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:34:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.975261+0000 mon.a (mon.0) 1185 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.976014+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.977352+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.978363+0000 mon.a (mon.0) 1188 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.979348+0000 mon.a (mon.0) 1189 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.980453+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.981233+0000 mon.a (mon.0) 1191 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.981758+0000 mon.a (mon.0) 1192 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.982766+0000 mon.a (mon.0) 1193 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:34:35.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.983201+0000 mon.a (mon.0) 1194 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:34:35.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.983654+0000 mon.a (mon.0) 1195 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:34:35.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.984207+0000 mon.a (mon.0) 1196 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:34:35.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.984861+0000 mon.a (mon.0) 1197 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:34:35.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:35 smithi138 bash[23952]: audit 2024-07-28T19:34:34.985953+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:34:35.706 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: [28/Jul/2024:19:34:35] ENGINE Bus STARTING 2024-07-28T19:34:35.706 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.447+0000 7f5301ce1640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.706 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.447+0000 7f5301ce1640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.706 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.447+0000 7f5301ce1640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.706 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.447+0000 7f5301ce1640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.706 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.447+0000 7f5301ce1640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.706 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.451+0000 7f52fe4da640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.707 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.451+0000 7f52fe4da640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.707 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.451+0000 7f52fe4da640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.707 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.451+0000 7f52fe4da640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.707 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: debug 2024-07-28T19:34:35.451+0000 7f52fe4da640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:34:35.707 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: [28/Jul/2024:19:34:35] ENGINE Serving on http://:::9283 2024-07-28T19:34:35.707 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:35 smithi100 bash[73305]: [28/Jul/2024:19:34:35] ENGINE Bus STARTED 2024-07-28T19:34:35.982 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 41943040 2024-07-28T19:34:36.017 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:36.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[23952]: cluster 2024-07-28T19:34:35.323299+0000 mon.a (mon.0) 1199 : cluster [INF] Manager daemon y is now available 2024-07-28T19:34:36.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[23952]: audit 2024-07-28T19:34:35.348729+0000 mon.a (mon.0) 1200 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:36.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[23952]: audit 2024-07-28T19:34:35.380975+0000 mon.a (mon.0) 1201 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:36.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[23952]: audit 2024-07-28T19:34:35.389266+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:34:36.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[23952]: audit 2024-07-28T19:34:35.427556+0000 mon.a (mon.0) 1203 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:34:36.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[23952]: cluster 2024-07-28T19:34:35.966051+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-07-28T19:34:36.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[21828]: cluster 2024-07-28T19:34:35.323299+0000 mon.a (mon.0) 1199 : cluster [INF] Manager daemon y is now available 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[21828]: audit 2024-07-28T19:34:35.348729+0000 mon.a (mon.0) 1200 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[21828]: audit 2024-07-28T19:34:35.380975+0000 mon.a (mon.0) 1201 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[21828]: audit 2024-07-28T19:34:35.389266+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[21828]: audit 2024-07-28T19:34:35.427556+0000 mon.a (mon.0) 1203 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[21828]: cluster 2024-07-28T19:34:35.966051+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[31715]: cluster 2024-07-28T19:34:35.323299+0000 mon.a (mon.0) 1199 : cluster [INF] Manager daemon y is now available 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[31715]: audit 2024-07-28T19:34:35.348729+0000 mon.a (mon.0) 1200 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[31715]: audit 2024-07-28T19:34:35.380975+0000 mon.a (mon.0) 1201 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[31715]: audit 2024-07-28T19:34:35.389266+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:34:36.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[31715]: audit 2024-07-28T19:34:35.427556+0000 mon.a (mon.0) 1203 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:34:36.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:36 smithi100 bash[31715]: cluster 2024-07-28T19:34:35.966051+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-07-28T19:34:36.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[52402]: debug 2024-07-28T19:34:36.418+0000 7ff7cf3c2040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:34:36.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[52402]: debug 2024-07-28T19:34:36.486+0000 7ff7cf3c2040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:34:36.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:36 smithi138 bash[52402]: debug 2024-07-28T19:34:36.638+0000 7ff7cf3c2040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:34:37.220 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:37.244 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:34:37.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[23952]: cluster 2024-07-28T19:34:36.048767+0000 mon.a (mon.0) 1205 : cluster [DBG] mgrmap e29: y(active, since 1.09384s) 2024-07-28T19:34:37.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[23952]: cluster 2024-07-28T19:34:36.077769+0000 mgr.y (mgr.15207) 1 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:37.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[23952]: audit 2024-07-28T19:34:36.999914+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.100:0/1392929945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:37.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[21828]: cluster 2024-07-28T19:34:36.048767+0000 mon.a (mon.0) 1205 : cluster [DBG] mgrmap e29: y(active, since 1.09384s) 2024-07-28T19:34:37.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[21828]: cluster 2024-07-28T19:34:36.077769+0000 mgr.y (mgr.15207) 1 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:37.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[21828]: audit 2024-07-28T19:34:36.999914+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.100:0/1392929945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:37.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[31715]: cluster 2024-07-28T19:34:36.048767+0000 mon.a (mon.0) 1205 : cluster [DBG] mgrmap e29: y(active, since 1.09384s) 2024-07-28T19:34:37.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[31715]: cluster 2024-07-28T19:34:36.077769+0000 mgr.y (mgr.15207) 1 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:37.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[31715]: audit 2024-07-28T19:34:36.999914+0000 mon.c (mon.2) 191 : audit [DBG] from='client.? 172.21.15.100:0/1392929945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:37.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: debug 2024-07-28T19:34:37.275+0000 7f52eb4b4640 -1 log_channel(cephadm) log [ERR] : [28/Jul/2024:19:34:37] ENGINE Error in HTTPServer.serve 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: Traceback (most recent call last): 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: self._connections.run(self.expiration_interval) 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: self._run(expiration_interval) 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: s = self.context.wrap_socket( 2024-07-28T19:34:37.495 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:34:37.496 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: return self.sslsocket_class._create( 2024-07-28T19:34:37.496 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:34:37.496 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: self.do_handshake() 2024-07-28T19:34:37.496 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:34:37.496 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: self._sslobj.do_handshake() 2024-07-28T19:34:37.496 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:34:37 smithi100 bash[73305]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:34:37.650 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[52402]: debug 2024-07-28T19:34:37.314+0000 7ff7cf3c2040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:34:37.650 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[52402]: debug 2024-07-28T19:34:37.390+0000 7ff7cf3c2040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:34:37.650 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[52402]: debug 2024-07-28T19:34:37.462+0000 7ff7cf3c2040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:34:37.650 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[52402]: debug 2024-07-28T19:34:37.650+0000 7ff7cf3c2040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:34:38.040 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[52402]: debug 2024-07-28T19:34:37.734+0000 7ff7cf3c2040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:34:38.040 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:37 smithi138 bash[52402]: debug 2024-07-28T19:34:37.802+0000 7ff7cf3c2040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:34:38.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[52402]: debug 2024-07-28T19:34:38.074+0000 7ff7cf3c2040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:34:38.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[52402]: debug 2024-07-28T19:34:38.146+0000 7ff7cf3c2040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:34:38.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: cluster 2024-07-28T19:34:37.049413+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-07-28T19:34:38.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: cluster 2024-07-28T19:34:37.049871+0000 mon.a (mon.0) 1207 : cluster [DBG] mgrmap e30: y(active, since 2s) 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: cephadm 2024-07-28T19:34:37.155252+0000 mgr.y (mgr.15207) 3 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Bus STARTING 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: cephadm 2024-07-28T19:34:37.277351+0000 mgr.y (mgr.15207) 4 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: cephadm 2024-07-28T19:34:37.280808+0000 mgr.y (mgr.15207) 5 : cephadm [ERR] [28/Jul/2024:19:34:37] ENGINE Error in HTTPServer.serve 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: Traceback (most recent call last): 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: self._connections.run(self.expiration_interval) 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: self._run(expiration_interval) 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:34:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: s = self.context.wrap_socket( 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: return self.sslsocket_class._create( 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: self.do_handshake() 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: self._sslobj.do_handshake() 2024-07-28T19:34:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:34:38.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: cephadm 2024-07-28T19:34:37.379398+0000 mgr.y (mgr.15207) 6 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Serving on http://172.21.15.100:8765 2024-07-28T19:34:38.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: cephadm 2024-07-28T19:34:37.379573+0000 mgr.y (mgr.15207) 7 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Bus STARTED 2024-07-28T19:34:38.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[23952]: cluster 2024-07-28T19:34:37.899173+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-07-28T19:34:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: cluster 2024-07-28T19:34:37.049413+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-07-28T19:34:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: cluster 2024-07-28T19:34:37.049871+0000 mon.a (mon.0) 1207 : cluster [DBG] mgrmap e30: y(active, since 2s) 2024-07-28T19:34:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: cephadm 2024-07-28T19:34:37.155252+0000 mgr.y (mgr.15207) 3 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Bus STARTING 2024-07-28T19:34:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: cephadm 2024-07-28T19:34:37.277351+0000 mgr.y (mgr.15207) 4 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: cephadm 2024-07-28T19:34:37.280808+0000 mgr.y (mgr.15207) 5 : cephadm [ERR] [28/Jul/2024:19:34:37] ENGINE Error in HTTPServer.serve 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: Traceback (most recent call last): 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: self._connections.run(self.expiration_interval) 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: self._run(expiration_interval) 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: s = self.context.wrap_socket( 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:34:38.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: return self.sslsocket_class._create( 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: self.do_handshake() 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: self._sslobj.do_handshake() 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: cephadm 2024-07-28T19:34:37.379398+0000 mgr.y (mgr.15207) 6 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Serving on http://172.21.15.100:8765 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: cephadm 2024-07-28T19:34:37.379573+0000 mgr.y (mgr.15207) 7 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Bus STARTED 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[21828]: cluster 2024-07-28T19:34:37.899173+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: cluster 2024-07-28T19:34:37.049413+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: cluster 2024-07-28T19:34:37.049871+0000 mon.a (mon.0) 1207 : cluster [DBG] mgrmap e30: y(active, since 2s) 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: cephadm 2024-07-28T19:34:37.155252+0000 mgr.y (mgr.15207) 3 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Bus STARTING 2024-07-28T19:34:38.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: cephadm 2024-07-28T19:34:37.277351+0000 mgr.y (mgr.15207) 4 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: cephadm 2024-07-28T19:34:37.280808+0000 mgr.y (mgr.15207) 5 : cephadm [ERR] [28/Jul/2024:19:34:37] ENGINE Error in HTTPServer.serve 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: Traceback (most recent call last): 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: self._connections.run(self.expiration_interval) 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: self._run(expiration_interval) 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: s = self.context.wrap_socket( 2024-07-28T19:34:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: return self.sslsocket_class._create( 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: self.do_handshake() 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: self._sslobj.do_handshake() 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: cephadm 2024-07-28T19:34:37.379398+0000 mgr.y (mgr.15207) 6 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Serving on http://172.21.15.100:8765 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: cephadm 2024-07-28T19:34:37.379573+0000 mgr.y (mgr.15207) 7 : cephadm [INF] [28/Jul/2024:19:34:37] ENGINE Bus STARTED 2024-07-28T19:34:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:38 smithi100 bash[31715]: cluster 2024-07-28T19:34:37.899173+0000 mon.a (mon.0) 1208 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-07-28T19:34:38.653 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[52402]: debug 2024-07-28T19:34:38.346+0000 7ff7cf3c2040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:34:38.653 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[52402]: debug 2024-07-28T19:34:38.422+0000 7ff7cf3c2040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:34:38.914 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 41943040~10 2024-07-28T19:34:38.927 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap4 2024-07-28T19:34:39.037 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[52402]: debug 2024-07-28T19:34:38.682+0000 7ff7cf3c2040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-28T19:34:39.037 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[52402]: debug 2024-07-28T19:34:38.862+0000 7ff7cf3c2040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:34:39.038 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:38 smithi138 bash[58836]: ts=2024-07-28T19:34:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:39.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:39 smithi138 bash[52402]: debug 2024-07-28T19:34:39.038+0000 7ff7cf3c2040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:34:39.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:39 smithi138 bash[52402]: debug 2024-07-28T19:34:39.122+0000 7ff7cf3c2040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:34:39.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:39 smithi138 bash[52402]: debug 2024-07-28T19:34:39.306+0000 7ff7cf3c2040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:34:40.219 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:39 smithi138 bash[23952]: cluster 2024-07-28T19:34:38.902745+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-07-28T19:34:40.219 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:39 smithi138 bash[23952]: cluster 2024-07-28T19:34:38.982271+0000 mgr.y (mgr.15207) 8 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:40.219 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:39 smithi138 bash[23952]: cluster 2024-07-28T19:34:39.053661+0000 mon.a (mon.0) 1210 : cluster [DBG] mgrmap e31: y(active, since 4s) 2024-07-28T19:34:40.219 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:39 smithi138 bash[23952]: audit 2024-07-28T19:34:39.358843+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.100:0/1717185567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:40.219 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:40 smithi138 bash[52402]: debug 2024-07-28T19:34:40.138+0000 7ff7cf3c2040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:34:40.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:39 smithi100 bash[21828]: cluster 2024-07-28T19:34:38.902745+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-07-28T19:34:40.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:39 smithi100 bash[21828]: cluster 2024-07-28T19:34:38.982271+0000 mgr.y (mgr.15207) 8 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:40.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:39 smithi100 bash[21828]: cluster 2024-07-28T19:34:39.053661+0000 mon.a (mon.0) 1210 : cluster [DBG] mgrmap e31: y(active, since 4s) 2024-07-28T19:34:40.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:39 smithi100 bash[21828]: audit 2024-07-28T19:34:39.358843+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.100:0/1717185567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:40.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:39 smithi100 bash[31715]: cluster 2024-07-28T19:34:38.902745+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-07-28T19:34:40.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:39 smithi100 bash[31715]: cluster 2024-07-28T19:34:38.982271+0000 mgr.y (mgr.15207) 8 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:40.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:39 smithi100 bash[31715]: cluster 2024-07-28T19:34:39.053661+0000 mon.a (mon.0) 1210 : cluster [DBG] mgrmap e31: y(active, since 4s) 2024-07-28T19:34:40.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:39 smithi100 bash[31715]: audit 2024-07-28T19:34:39.358843+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.100:0/1717185567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:40.524 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:40 smithi138 bash[52402]: debug 2024-07-28T19:34:40.218+0000 7ff7cf3c2040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:34:40.524 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:40 smithi138 bash[52402]: debug 2024-07-28T19:34:40.450+0000 7ff7cf3c2040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:34:40.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:40 smithi138 bash[52402]: debug 2024-07-28T19:34:40.526+0000 7ff7cf3c2040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:34:40.916 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 41943040 2024-07-28T19:34:40.936 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:34:41.020 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:34:41.178 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:40 smithi138 bash[23952]: cluster 2024-07-28T19:34:39.913650+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-07-28T19:34:41.178 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:40 smithi138 bash[52402]: debug 2024-07-28T19:34:40.950+0000 7ff7cf3c2040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:34:41.178 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[52402]: debug 2024-07-28T19:34:41.054+0000 7ff7cf3c2040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:34:41.178 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[52402]: [28/Jul/2024:19:34:41] ENGINE Bus STARTING 2024-07-28T19:34:41.178 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[52402]: CherryPy Checker: 2024-07-28T19:34:41.178 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[52402]: The Application mounted at '' has an empty config. 2024-07-28T19:34:41.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:40 smithi100 bash[21828]: cluster 2024-07-28T19:34:39.913650+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-07-28T19:34:41.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:40 smithi100 bash[31715]: cluster 2024-07-28T19:34:39.913650+0000 mon.a (mon.0) 1211 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-07-28T19:34:41.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[52402]: [28/Jul/2024:19:34:41] ENGINE Serving on http://:::9283 2024-07-28T19:34:41.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[52402]: [28/Jul/2024:19:34:41] ENGINE Bus STARTED 2024-07-28T19:34:42.148 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (7934 ms) 2024-07-28T19:34:42.148 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-07-28T19:34:42.200 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:42.221 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: cluster 2024-07-28T19:34:40.913072+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: cluster 2024-07-28T19:34:40.982859+0000 mgr.y (mgr.15207) 9 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: cluster 2024-07-28T19:34:41.079547+0000 mon.a (mon.0) 1213 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.079752+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.080800+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.083592+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.083967+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.356810+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.360467+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.573792+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.582926+0000 mon.a (mon.0) 1217 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[21828]: audit 2024-07-28T19:34:41.718432+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.100:0/3405538864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: cluster 2024-07-28T19:34:40.913072+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-07-28T19:34:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: cluster 2024-07-28T19:34:40.982859+0000 mgr.y (mgr.15207) 9 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: cluster 2024-07-28T19:34:41.079547+0000 mon.a (mon.0) 1213 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:34:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.079752+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:34:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.080800+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:34:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.083592+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:34:42.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.083967+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:34:42.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.356810+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.360467+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.573792+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.582926+0000 mon.a (mon.0) 1217 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:41 smithi100 bash[31715]: audit 2024-07-28T19:34:41.718432+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.100:0/3405538864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: cluster 2024-07-28T19:34:40.913072+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-07-28T19:34:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: cluster 2024-07-28T19:34:40.982859+0000 mgr.y (mgr.15207) 9 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: cluster 2024-07-28T19:34:41.079547+0000 mon.a (mon.0) 1213 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:34:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.079752+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:34:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.080800+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:34:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.083592+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:34:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.083967+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.138:0/950409544' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:34:42.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.356810+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.360467+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.573792+0000 mon.a (mon.0) 1216 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.582926+0000 mon.a (mon.0) 1217 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:42.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:41 smithi138 bash[23952]: audit 2024-07-28T19:34:41.718432+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.100:0/3405538864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:42.588 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.AbortInUseImage (440 ms) 2024-07-28T19:34:42.588 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-07-28T19:34:42.667 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:42.724 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:34:42.792 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:34:42.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:42 smithi138 bash[58836]: ts=2024-07-28T19:34:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:42.870 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:43.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[21828]: cluster 2024-07-28T19:34:41.934854+0000 mon.a (mon.0) 1218 : cluster [DBG] mgrmap e32: y(active, since 6s), standbys: x 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[21828]: cluster 2024-07-28T19:34:41.935047+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[21828]: audit 2024-07-28T19:34:41.936345+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[21828]: audit 2024-07-28T19:34:42.140588+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[21828]: audit 2024-07-28T19:34:42.146558+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[21828]: audit 2024-07-28T19:34:42.297359+0000 mon.a (mon.0) 1223 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[21828]: audit 2024-07-28T19:34:42.307970+0000 mon.a (mon.0) 1224 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[31715]: cluster 2024-07-28T19:34:41.934854+0000 mon.a (mon.0) 1218 : cluster [DBG] mgrmap e32: y(active, since 6s), standbys: x 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[31715]: cluster 2024-07-28T19:34:41.935047+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-07-28T19:34:43.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[31715]: audit 2024-07-28T19:34:41.936345+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:34:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[31715]: audit 2024-07-28T19:34:42.140588+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[31715]: audit 2024-07-28T19:34:42.146558+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[31715]: audit 2024-07-28T19:34:42.297359+0000 mon.a (mon.0) 1223 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:42 smithi100 bash[31715]: audit 2024-07-28T19:34:42.307970+0000 mon.a (mon.0) 1224 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:42 smithi138 bash[23952]: cluster 2024-07-28T19:34:41.934854+0000 mon.a (mon.0) 1218 : cluster [DBG] mgrmap e32: y(active, since 6s), standbys: x 2024-07-28T19:34:43.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:42 smithi138 bash[23952]: cluster 2024-07-28T19:34:41.935047+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-07-28T19:34:43.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:42 smithi138 bash[23952]: audit 2024-07-28T19:34:41.936345+0000 mon.a (mon.0) 1220 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:34:43.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:42 smithi138 bash[23952]: audit 2024-07-28T19:34:42.140588+0000 mon.a (mon.0) 1221 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:42 smithi138 bash[23952]: audit 2024-07-28T19:34:42.146558+0000 mon.a (mon.0) 1222 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:42 smithi138 bash[23952]: audit 2024-07-28T19:34:42.297359+0000 mon.a (mon.0) 1223 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:42 smithi138 bash[23952]: audit 2024-07-28T19:34:42.307970+0000 mon.a (mon.0) 1224 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:43.320 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:34:43.385 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:34:43.488 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:34:43.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:43 smithi138 bash[58836]: ts=2024-07-28T19:34:43.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:43.840 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (1253 ms) 2024-07-28T19:34:43.840 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-07-28T19:34:43.890 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:43.912 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:34:44.007 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:34:44.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:43 smithi100 bash[21828]: cluster 2024-07-28T19:34:42.983700+0000 mgr.y (mgr.15207) 10 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 18 KiB/s wr, 314 op/s 2024-07-28T19:34:44.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:43 smithi100 bash[31715]: cluster 2024-07-28T19:34:42.983700+0000 mgr.y (mgr.15207) 10 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 18 KiB/s wr, 314 op/s 2024-07-28T19:34:44.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:43 smithi138 bash[23952]: cluster 2024-07-28T19:34:42.983700+0000 mgr.y (mgr.15207) 10 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 270 KiB/s rd, 18 KiB/s wr, 314 op/s 2024-07-28T19:34:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:44 smithi100 bash[21828]: audit 2024-07-28T19:34:44.103673+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.100:0/4113571766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:44 smithi100 bash[31715]: audit 2024-07-28T19:34:44.103673+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.100:0/4113571766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:45.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:44 smithi138 bash[23952]: audit 2024-07-28T19:34:44.103673+0000 mon.a (mon.0) 1225 : audit [DBG] from='client.? 172.21.15.100:0/4113571766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:45.976 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 8388609~8388609 2024-07-28T19:34:46.025 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 0~1000 2024-07-28T19:34:46.025 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1010~1000 2024-07-28T19:34:46.025 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:34:46.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:45 smithi100 bash[21828]: cluster 2024-07-28T19:34:44.947689+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-07-28T19:34:46.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:45 smithi100 bash[21828]: cluster 2024-07-28T19:34:44.984523+0000 mgr.y (mgr.15207) 11 : cluster [DBG] pgmap v16: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 14 KiB/s wr, 253 op/s 2024-07-28T19:34:46.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:45 smithi100 bash[31715]: cluster 2024-07-28T19:34:44.947689+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-07-28T19:34:46.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:45 smithi100 bash[31715]: cluster 2024-07-28T19:34:44.984523+0000 mgr.y (mgr.15207) 11 : cluster [DBG] pgmap v16: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 14 KiB/s wr, 253 op/s 2024-07-28T19:34:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:45 smithi138 bash[23952]: cluster 2024-07-28T19:34:44.947689+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-07-28T19:34:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:45 smithi138 bash[23952]: cluster 2024-07-28T19:34:44.984523+0000 mgr.y (mgr.15207) 11 : cluster [DBG] pgmap v16: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 217 KiB/s rd, 14 KiB/s wr, 253 op/s 2024-07-28T19:34:47.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:46 smithi100 bash[21828]: cluster 2024-07-28T19:34:45.960342+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-07-28T19:34:47.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:46 smithi100 bash[21828]: audit 2024-07-28T19:34:46.460985+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.100:0/2128699319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:47.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:46 smithi100 bash[31715]: cluster 2024-07-28T19:34:45.960342+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-07-28T19:34:47.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:46 smithi100 bash[31715]: audit 2024-07-28T19:34:46.460985+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.100:0/2128699319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:47.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:46 smithi138 bash[23952]: cluster 2024-07-28T19:34:45.960342+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-07-28T19:34:47.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:46 smithi138 bash[23952]: audit 2024-07-28T19:34:46.460985+0000 mon.c (mon.2) 194 : audit [DBG] from='client.? 172.21.15.100:0/2128699319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:48.009 INFO:tasks.workunit.client.0.smithi100.stdout:write: X 8388609~1000 2024-07-28T19:34:48.030 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16777228~1000 2024-07-28T19:34:48.030 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 167772160 2024-07-28T19:34:48.047 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 83886079~8388609 2024-07-28T19:34:48.077 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:34:48.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:47 smithi100 bash[21828]: cluster 2024-07-28T19:34:46.965696+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-07-28T19:34:48.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:47 smithi100 bash[21828]: cluster 2024-07-28T19:34:46.985356+0000 mgr.y (mgr.15207) 12 : cluster [DBG] pgmap v19: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 23 MiB/s wr, 385 op/s 2024-07-28T19:34:48.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:47 smithi100 bash[31715]: cluster 2024-07-28T19:34:46.965696+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-07-28T19:34:48.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:47 smithi100 bash[31715]: cluster 2024-07-28T19:34:46.985356+0000 mgr.y (mgr.15207) 12 : cluster [DBG] pgmap v19: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 23 MiB/s wr, 385 op/s 2024-07-28T19:34:48.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:47 smithi138 bash[23952]: cluster 2024-07-28T19:34:46.965696+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-07-28T19:34:48.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:47 smithi138 bash[23952]: cluster 2024-07-28T19:34:46.985356+0000 mgr.y (mgr.15207) 12 : cluster [DBG] pgmap v19: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 23 MiB/s wr, 385 op/s 2024-07-28T19:34:49.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:48 smithi100 bash[21828]: cluster 2024-07-28T19:34:47.990797+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-07-28T19:34:49.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:48 smithi100 bash[21828]: audit 2024-07-28T19:34:48.815657+0000 mon.a (mon.0) 1230 : audit [DBG] from='client.? 172.21.15.100:0/1554759860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:49.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:48 smithi100 bash[31715]: cluster 2024-07-28T19:34:47.990797+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-07-28T19:34:49.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:48 smithi100 bash[31715]: audit 2024-07-28T19:34:48.815657+0000 mon.a (mon.0) 1230 : audit [DBG] from='client.? 172.21.15.100:0/1554759860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:49.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:48 smithi138 bash[23952]: cluster 2024-07-28T19:34:47.990797+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-07-28T19:34:49.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:48 smithi138 bash[23952]: audit 2024-07-28T19:34:48.815657+0000 mon.a (mon.0) 1230 : audit [DBG] from='client.? 172.21.15.100:0/1554759860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:50.019 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:50.071 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83886079~1 2024-07-28T19:34:50.088 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:34:50.253 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:50 smithi100 bash[21828]: cluster 2024-07-28T19:34:48.986797+0000 mgr.y (mgr.15207) 13 : cluster [DBG] pgmap v22: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 29 MiB/s wr, 488 op/s 2024-07-28T19:34:50.253 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:50 smithi100 bash[21828]: cluster 2024-07-28T19:34:48.987655+0000 mon.a (mon.0) 1231 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-07-28T19:34:50.254 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:50 smithi100 bash[31715]: cluster 2024-07-28T19:34:48.986797+0000 mgr.y (mgr.15207) 13 : cluster [DBG] pgmap v22: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 29 MiB/s wr, 488 op/s 2024-07-28T19:34:50.254 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:50 smithi100 bash[31715]: cluster 2024-07-28T19:34:48.987655+0000 mon.a (mon.0) 1231 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-07-28T19:34:50.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:50 smithi138 bash[23952]: cluster 2024-07-28T19:34:48.986797+0000 mgr.y (mgr.15207) 13 : cluster [DBG] pgmap v22: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 29 MiB/s wr, 488 op/s 2024-07-28T19:34:50.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:50 smithi138 bash[23952]: cluster 2024-07-28T19:34:48.987655+0000 mon.a (mon.0) 1231 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-07-28T19:34:51.005 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:34:50 smithi138 bash[52402]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:34:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-07-28T19:34:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:51 smithi138 bash[23952]: cluster 2024-07-28T19:34:50.000177+0000 mon.a (mon.0) 1232 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-07-28T19:34:51.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:51 smithi138 bash[23952]: audit 2024-07-28T19:34:50.379550+0000 mon.a (mon.0) 1233 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:51.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:51 smithi100 bash[21828]: cluster 2024-07-28T19:34:50.000177+0000 mon.a (mon.0) 1232 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-07-28T19:34:51.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:51 smithi100 bash[21828]: audit 2024-07-28T19:34:50.379550+0000 mon.a (mon.0) 1233 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:51.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:51 smithi100 bash[31715]: cluster 2024-07-28T19:34:50.000177+0000 mon.a (mon.0) 1232 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-07-28T19:34:51.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:51 smithi100 bash[31715]: audit 2024-07-28T19:34:50.379550+0000 mon.a (mon.0) 1233 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:34:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:52 smithi138 bash[23952]: cluster 2024-07-28T19:34:50.987337+0000 mgr.y (mgr.15207) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:52 smithi138 bash[23952]: cluster 2024-07-28T19:34:51.017360+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-07-28T19:34:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:52 smithi138 bash[23952]: audit 2024-07-28T19:34:51.179381+0000 mon.a (mon.0) 1235 : audit [DBG] from='client.? 172.21.15.100:0/2925181464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:52 smithi138 bash[23952]: audit 2024-07-28T19:34:51.776366+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:52 smithi138 bash[23952]: audit 2024-07-28T19:34:51.786085+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:52.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:52 smithi138 bash[23952]: audit 2024-07-28T19:34:51.789097+0000 mon.a (mon.0) 1238 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:52 smithi138 bash[23952]: audit 2024-07-28T19:34:51.790601+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:34:52.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[31715]: cluster 2024-07-28T19:34:50.987337+0000 mgr.y (mgr.15207) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[31715]: cluster 2024-07-28T19:34:51.017360+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[31715]: audit 2024-07-28T19:34:51.179381+0000 mon.a (mon.0) 1235 : audit [DBG] from='client.? 172.21.15.100:0/2925181464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[31715]: audit 2024-07-28T19:34:51.776366+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[31715]: audit 2024-07-28T19:34:51.786085+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[31715]: audit 2024-07-28T19:34:51.789097+0000 mon.a (mon.0) 1238 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[31715]: audit 2024-07-28T19:34:51.790601+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[21828]: cluster 2024-07-28T19:34:50.987337+0000 mgr.y (mgr.15207) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[21828]: cluster 2024-07-28T19:34:51.017360+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[21828]: audit 2024-07-28T19:34:51.179381+0000 mon.a (mon.0) 1235 : audit [DBG] from='client.? 172.21.15.100:0/2925181464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[21828]: audit 2024-07-28T19:34:51.776366+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:52.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[21828]: audit 2024-07-28T19:34:51.786085+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:52.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[21828]: audit 2024-07-28T19:34:51.789097+0000 mon.a (mon.0) 1238 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:34:52.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:52 smithi100 bash[21828]: audit 2024-07-28T19:34:51.790601+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:34:52.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:52 smithi138 bash[58836]: ts=2024-07-28T19:34:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:51.792479+0000 mgr.y (mgr.15207) 15 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:51.793430+0000 mgr.y (mgr.15207) 16 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:51.957136+0000 mgr.y (mgr.15207) 17 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:51.957306+0000 mgr.y (mgr.15207) 18 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cluster 2024-07-28T19:34:52.026519+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:52.094094+0000 mgr.y (mgr.15207) 19 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:52.102167+0000 mgr.y (mgr.15207) 20 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:52.216929+0000 mgr.y (mgr.15207) 21 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:52.238057+0000 mgr.y (mgr.15207) 22 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: audit 2024-07-28T19:34:52.352175+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: audit 2024-07-28T19:34:52.357592+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: audit 2024-07-28T19:34:52.374452+0000 mon.a (mon.0) 1243 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: audit 2024-07-28T19:34:52.380825+0000 mon.a (mon.0) 1244 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: audit 2024-07-28T19:34:52.389632+0000 mon.a (mon.0) 1245 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cephadm 2024-07-28T19:34:52.428862+0000 mgr.y (mgr.15207) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:34:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[23952]: cluster 2024-07-28T19:34:52.909145+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:51.792479+0000 mgr.y (mgr.15207) 15 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:51.793430+0000 mgr.y (mgr.15207) 16 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:51.957136+0000 mgr.y (mgr.15207) 17 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:51.957306+0000 mgr.y (mgr.15207) 18 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cluster 2024-07-28T19:34:52.026519+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:52.094094+0000 mgr.y (mgr.15207) 19 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:52.102167+0000 mgr.y (mgr.15207) 20 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:52.216929+0000 mgr.y (mgr.15207) 21 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:52.238057+0000 mgr.y (mgr.15207) 22 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: audit 2024-07-28T19:34:52.352175+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: audit 2024-07-28T19:34:52.357592+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: audit 2024-07-28T19:34:52.374452+0000 mon.a (mon.0) 1243 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: audit 2024-07-28T19:34:52.380825+0000 mon.a (mon.0) 1244 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: audit 2024-07-28T19:34:52.389632+0000 mon.a (mon.0) 1245 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cephadm 2024-07-28T19:34:52.428862+0000 mgr.y (mgr.15207) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[21828]: cluster 2024-07-28T19:34:52.909145+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:51.792479+0000 mgr.y (mgr.15207) 15 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:51.793430+0000 mgr.y (mgr.15207) 16 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:51.957136+0000 mgr.y (mgr.15207) 17 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:51.957306+0000 mgr.y (mgr.15207) 18 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cluster 2024-07-28T19:34:52.026519+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:52.094094+0000 mgr.y (mgr.15207) 19 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:34:53.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:52.102167+0000 mgr.y (mgr.15207) 20 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:52.216929+0000 mgr.y (mgr.15207) 21 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:52.238057+0000 mgr.y (mgr.15207) 22 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: audit 2024-07-28T19:34:52.352175+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: audit 2024-07-28T19:34:52.357592+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: audit 2024-07-28T19:34:52.374452+0000 mon.a (mon.0) 1243 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: audit 2024-07-28T19:34:52.380825+0000 mon.a (mon.0) 1244 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: audit 2024-07-28T19:34:52.389632+0000 mon.a (mon.0) 1245 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cephadm 2024-07-28T19:34:52.428862+0000 mgr.y (mgr.15207) 23 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-07-28T19:34:53.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:53 smithi100 bash[31715]: cluster 2024-07-28T19:34:52.909145+0000 mon.a (mon.0) 1246 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-07-28T19:34:53.687 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:34:53.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:53 smithi138 bash[58836]: ts=2024-07-28T19:34:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:34:53.852 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:34:54.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:54 smithi138 bash[23952]: cephadm 2024-07-28T19:34:52.710282+0000 mgr.y (mgr.15207) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:34:54.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:54 smithi138 bash[23952]: cluster 2024-07-28T19:34:52.988189+0000 mgr.y (mgr.15207) 25 : cluster [DBG] pgmap v28: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-07-28T19:34:54.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:54 smithi138 bash[23952]: audit 2024-07-28T19:34:53.554674+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.100:0/2211200434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:54.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:54 smithi100 bash[21828]: cephadm 2024-07-28T19:34:52.710282+0000 mgr.y (mgr.15207) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:34:54.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:54 smithi100 bash[21828]: cluster 2024-07-28T19:34:52.988189+0000 mgr.y (mgr.15207) 25 : cluster [DBG] pgmap v28: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-07-28T19:34:54.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:54 smithi100 bash[21828]: audit 2024-07-28T19:34:53.554674+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.100:0/2211200434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:54.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:54 smithi100 bash[31715]: cephadm 2024-07-28T19:34:52.710282+0000 mgr.y (mgr.15207) 24 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi138 2024-07-28T19:34:54.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:54 smithi100 bash[31715]: cluster 2024-07-28T19:34:52.988189+0000 mgr.y (mgr.15207) 25 : cluster [DBG] pgmap v28: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-07-28T19:34:54.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:54 smithi100 bash[31715]: audit 2024-07-28T19:34:53.554674+0000 mon.c (mon.2) 195 : audit [DBG] from='client.? 172.21.15.100:0/2211200434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:54.948 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 0~8388609 2024-07-28T19:34:54.994 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 16777218~8388609 2024-07-28T19:34:55.111 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:34:56.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:55 smithi100 bash[21828]: cluster 2024-07-28T19:34:54.924408+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-07-28T19:34:56.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:55 smithi100 bash[21828]: cluster 2024-07-28T19:34:54.990942+0000 mgr.y (mgr.15207) 26 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-07-28T19:34:56.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:55 smithi100 bash[31715]: cluster 2024-07-28T19:34:54.924408+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-07-28T19:34:56.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:55 smithi100 bash[31715]: cluster 2024-07-28T19:34:54.990942+0000 mgr.y (mgr.15207) 26 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-07-28T19:34:56.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:55 smithi138 bash[23952]: cluster 2024-07-28T19:34:54.924408+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-07-28T19:34:56.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:55 smithi138 bash[23952]: cluster 2024-07-28T19:34:54.990942+0000 mgr.y (mgr.15207) 26 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-07-28T19:34:57.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:56 smithi138 bash[23952]: audit 2024-07-28T19:34:55.923737+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.100:0/159276685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:57.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:56 smithi138 bash[23952]: cluster 2024-07-28T19:34:55.929774+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-07-28T19:34:57.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:56 smithi100 bash[21828]: audit 2024-07-28T19:34:55.923737+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.100:0/159276685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:57.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:56 smithi100 bash[21828]: cluster 2024-07-28T19:34:55.929774+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-07-28T19:34:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:56 smithi100 bash[31715]: audit 2024-07-28T19:34:55.923737+0000 mon.a (mon.0) 1248 : audit [DBG] from='client.? 172.21.15.100:0/159276685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:56 smithi100 bash[31715]: cluster 2024-07-28T19:34:55.929774+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-07-28T19:34:57.554 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 systemd[1]: Stopping Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.421Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.422Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-07-28T19:34:57.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[58836]: ts=2024-07-28T19:34:57.423Z caller=main.go:1147 level=info msg="See you next time!" 2024-07-28T19:34:58.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:57 smithi100 bash[31715]: cluster 2024-07-28T19:34:56.933797+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-07-28T19:34:58.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:57 smithi100 bash[31715]: cluster 2024-07-28T19:34:56.992021+0000 mgr.y (mgr.15207) 27 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 41 MiB/s rd, 23 MiB/s wr, 367 op/s 2024-07-28T19:34:58.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:57 smithi100 bash[21828]: cluster 2024-07-28T19:34:56.933797+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-07-28T19:34:58.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:57 smithi100 bash[21828]: cluster 2024-07-28T19:34:56.992021+0000 mgr.y (mgr.15207) 27 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 41 MiB/s rd, 23 MiB/s wr, 367 op/s 2024-07-28T19:34:58.304 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[60525]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-prometheus-a 2024-07-28T19:34:58.304 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:58 smithi138 bash[60586]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-prometheus-a 2024-07-28T19:34:58.304 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:58 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-07-28T19:34:58.304 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:58 smithi138 systemd[1]: Stopped Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:58.304 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:34:58 smithi138 systemd[1]: Started Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:34:58.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[23952]: cluster 2024-07-28T19:34:56.933797+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-07-28T19:34:58.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:57 smithi138 bash[23952]: cluster 2024-07-28T19:34:56.992021+0000 mgr.y (mgr.15207) 27 : cluster [DBG] pgmap v33: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 41 MiB/s rd, 23 MiB/s wr, 367 op/s 2024-07-28T19:34:59.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:58 smithi138 bash[23952]: cluster 2024-07-28T19:34:57.945797+0000 mon.a (mon.0) 1251 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-07-28T19:34:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:58 smithi138 bash[23952]: audit 2024-07-28T19:34:58.198951+0000 mon.a (mon.0) 1252 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:58 smithi138 bash[23952]: audit 2024-07-28T19:34:58.206871+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:58 smithi138 bash[23952]: audit 2024-07-28T19:34:58.212467+0000 mon.a (mon.0) 1254 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:58 smithi138 bash[23952]: audit 2024-07-28T19:34:58.213474+0000 mgr.y (mgr.15207) 28 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:58 smithi138 bash[23952]: audit 2024-07-28T19:34:58.255162+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:58 smithi138 bash[23952]: audit 2024-07-28T19:34:58.289703+0000 mon.a (mon.0) 1256 : audit [DBG] from='client.? 172.21.15.100:0/3274635077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:59.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[31715]: cluster 2024-07-28T19:34:57.945797+0000 mon.a (mon.0) 1251 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-07-28T19:34:59.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[31715]: audit 2024-07-28T19:34:58.198951+0000 mon.a (mon.0) 1252 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[31715]: audit 2024-07-28T19:34:58.206871+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[31715]: audit 2024-07-28T19:34:58.212467+0000 mon.a (mon.0) 1254 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[31715]: audit 2024-07-28T19:34:58.213474+0000 mgr.y (mgr.15207) 28 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[31715]: audit 2024-07-28T19:34:58.255162+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[31715]: audit 2024-07-28T19:34:58.289703+0000 mon.a (mon.0) 1256 : audit [DBG] from='client.? 172.21.15.100:0/3274635077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[21828]: cluster 2024-07-28T19:34:57.945797+0000 mon.a (mon.0) 1251 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[21828]: audit 2024-07-28T19:34:58.198951+0000 mon.a (mon.0) 1252 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[21828]: audit 2024-07-28T19:34:58.206871+0000 mon.a (mon.0) 1253 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[21828]: audit 2024-07-28T19:34:58.212467+0000 mon.a (mon.0) 1254 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[21828]: audit 2024-07-28T19:34:58.213474+0000 mgr.y (mgr.15207) 28 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[21828]: audit 2024-07-28T19:34:58.255162+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:34:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:58 smithi100 bash[21828]: audit 2024-07-28T19:34:58.289703+0000 mon.a (mon.0) 1256 : audit [DBG] from='client.? 172.21.15.100:0/3274635077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:34:59.612 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.AbortWithSnapshots (15771 ms) 2024-07-28T19:34:59.612 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-07-28T19:34:59.687 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:34:59.714 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:34:59.736 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:00.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:59 smithi100 bash[21828]: cluster 2024-07-28T19:34:58.958214+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-07-28T19:35:00.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:34:59 smithi100 bash[21828]: cluster 2024-07-28T19:34:58.995306+0000 mgr.y (mgr.15207) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 374 op/s 2024-07-28T19:35:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:59 smithi100 bash[31715]: cluster 2024-07-28T19:34:58.958214+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-07-28T19:35:00.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:34:59 smithi100 bash[31715]: cluster 2024-07-28T19:34:58.995306+0000 mgr.y (mgr.15207) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 374 op/s 2024-07-28T19:35:00.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:59 smithi138 bash[23952]: cluster 2024-07-28T19:34:58.958214+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-07-28T19:35:00.250 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:34:59 smithi138 bash[23952]: cluster 2024-07-28T19:34:58.995306+0000 mgr.y (mgr.15207) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 374 op/s 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.282Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.283Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.283Z caller=main.go:570 level=info host_details="(Linux 5.15.0-117-generic #127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024 x86_64 smithi138 (none))" 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.283Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.283Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.288Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.289Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.290Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.290Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-07-28T19:35:00.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.291Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.291Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.701µs 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.291Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.292Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.329Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.336Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.336Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.336Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=60.959µs wal_replay_duration=44.97497ms wbl_replay_duration=133ns total_replay_duration=45.293601ms 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.340Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.340Z caller=main.go:1029 level=info msg="TSDB started" 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.340Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-07-28T19:35:00.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.367Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.660105ms db_storage=909ns remote_storage=1.212µs web_handler=530ns query_engine=662ns scrape=278.158µs scrape_sd=117.6µs notify=13.774µs notify_sd=14.316µs rules=25.808443ms tracing=7.922µs 2024-07-28T19:35:00.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.367Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-07-28T19:35:00.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[60615]: ts=2024-07-28T19:35:00.367Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-07-28T19:35:01.000 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:01.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[23952]: cluster 2024-07-28T19:34:59.966818+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-07-28T19:35:01.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:00 smithi138 bash[23952]: audit 2024-07-28T19:35:00.652999+0000 mon.a (mon.0) 1259 : audit [DBG] from='client.? 172.21.15.100:0/2606986410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:01.137 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:01.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:00 smithi100 bash[21828]: cluster 2024-07-28T19:34:59.966818+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-07-28T19:35:01.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:00 smithi100 bash[21828]: audit 2024-07-28T19:35:00.652999+0000 mon.a (mon.0) 1259 : audit [DBG] from='client.? 172.21.15.100:0/2606986410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:01.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:00 smithi100 bash[31715]: cluster 2024-07-28T19:34:59.966818+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-07-28T19:35:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:00 smithi100 bash[31715]: audit 2024-07-28T19:35:00.652999+0000 mon.a (mon.0) 1259 : audit [DBG] from='client.? 172.21.15.100:0/2606986410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:02.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[21828]: cluster 2024-07-28T19:35:00.982790+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[21828]: cluster 2024-07-28T19:35:00.996080+0000 mgr.y (mgr.15207) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[21828]: cephadm 2024-07-28T19:35:01.115388+0000 mgr.y (mgr.15207) 31 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[21828]: audit 2024-07-28T19:35:01.116149+0000 mon.a (mon.0) 1261 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[21828]: audit 2024-07-28T19:35:01.117793+0000 mon.a (mon.0) 1262 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[21828]: audit 2024-07-28T19:35:01.119705+0000 mon.a (mon.0) 1263 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[21828]: cephadm 2024-07-28T19:35:01.121541+0000 mgr.y (mgr.15207) 32 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[31715]: cluster 2024-07-28T19:35:00.982790+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[31715]: cluster 2024-07-28T19:35:00.996080+0000 mgr.y (mgr.15207) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:02.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[31715]: cephadm 2024-07-28T19:35:01.115388+0000 mgr.y (mgr.15207) 31 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-28T19:35:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[31715]: audit 2024-07-28T19:35:01.116149+0000 mon.a (mon.0) 1261 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:35:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[31715]: audit 2024-07-28T19:35:01.117793+0000 mon.a (mon.0) 1262 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:35:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[31715]: audit 2024-07-28T19:35:01.119705+0000 mon.a (mon.0) 1263 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:01 smithi100 bash[31715]: cephadm 2024-07-28T19:35:01.121541+0000 mgr.y (mgr.15207) 32 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:35:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:01 smithi138 bash[23952]: cluster 2024-07-28T19:35:00.982790+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-07-28T19:35:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:01 smithi138 bash[23952]: cluster 2024-07-28T19:35:00.996080+0000 mgr.y (mgr.15207) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:02.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:01 smithi138 bash[23952]: cephadm 2024-07-28T19:35:01.115388+0000 mgr.y (mgr.15207) 31 : cephadm [INF] Upgrade: Updating mgr.x 2024-07-28T19:35:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:01 smithi138 bash[23952]: audit 2024-07-28T19:35:01.116149+0000 mon.a (mon.0) 1261 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:35:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:01 smithi138 bash[23952]: audit 2024-07-28T19:35:01.117793+0000 mon.a (mon.0) 1262 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:35:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:01 smithi138 bash[23952]: audit 2024-07-28T19:35:01.119705+0000 mon.a (mon.0) 1263 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:01 smithi138 bash[23952]: cephadm 2024-07-28T19:35:01.121541+0000 mgr.y (mgr.15207) 32 : cephadm [INF] Deploying daemon mgr.x on smithi138 2024-07-28T19:35:02.474 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:35:02.628 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:02 smithi138 bash[60615]: ts=2024-07-28T19:35:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:02.673 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:35:03.213 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV1Parent (3601 ms) 2024-07-28T19:35:03.213 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-07-28T19:35:03.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:02 smithi100 bash[21828]: cluster 2024-07-28T19:35:01.988436+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-07-28T19:35:03.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:02 smithi100 bash[31715]: cluster 2024-07-28T19:35:01.988436+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-07-28T19:35:03.291 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:03.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:02 smithi138 bash[23952]: cluster 2024-07-28T19:35:01.988436+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-07-28T19:35:03.317 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:03.339 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:03.755 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:03.755 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:03.756 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:03.756 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:03.756 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:03.757 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:03 smithi138 bash[60615]: ts=2024-07-28T19:35:03.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:03.757 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:03.757 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:04.018 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:04 smithi138 bash[23952]: cluster 2024-07-28T19:35:02.991303+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-07-28T19:35:04.018 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:04 smithi138 bash[23952]: cluster 2024-07-28T19:35:02.996799+0000 mgr.y (mgr.15207) 33 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 12 KiB/s wr, 150 op/s 2024-07-28T19:35:04.018 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:04 smithi138 bash[23952]: audit 2024-07-28T19:35:03.025109+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.100:0/4046783241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:04.018 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:04.018 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: Stopping Ceph mgr.x for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:35:04.019 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:35:03 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:04.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:04 smithi100 bash[21828]: cluster 2024-07-28T19:35:02.991303+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-07-28T19:35:04.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:04 smithi100 bash[21828]: cluster 2024-07-28T19:35:02.996799+0000 mgr.y (mgr.15207) 33 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 12 KiB/s wr, 150 op/s 2024-07-28T19:35:04.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:04 smithi100 bash[21828]: audit 2024-07-28T19:35:03.025109+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.100:0/4046783241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:04.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:04 smithi100 bash[31715]: cluster 2024-07-28T19:35:02.991303+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-07-28T19:35:04.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:04 smithi100 bash[31715]: cluster 2024-07-28T19:35:02.996799+0000 mgr.y (mgr.15207) 33 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 12 KiB/s wr, 150 op/s 2024-07-28T19:35:04.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:04 smithi100 bash[31715]: audit 2024-07-28T19:35:03.025109+0000 mon.c (mon.2) 196 : audit [DBG] from='client.? 172.21.15.100:0/4046783241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:04.927 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:04 smithi138 bash[60897]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mgr-x 2024-07-28T19:35:04.928 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:04 smithi138 bash[60956]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mgr-x 2024-07-28T19:35:04.928 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:35:04.928 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-07-28T19:35:04.928 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: Stopped Ceph mgr.x for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:35:05.042 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:05.130 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:05.304 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.305 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.305 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.305 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:05 smithi138 bash[23952]: cluster 2024-07-28T19:35:04.013151+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-07-28T19:35:05.306 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.306 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:05 smithi138 systemd[1]: Started Ceph mgr.x for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:35:05.307 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.307 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:35:04 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:05.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:05 smithi100 bash[21828]: cluster 2024-07-28T19:35:04.013151+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-07-28T19:35:05.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:05 smithi100 bash[31715]: cluster 2024-07-28T19:35:04.013151+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-07-28T19:35:06.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[23952]: cluster 2024-07-28T19:35:04.997608+0000 mgr.y (mgr.15207) 34 : cluster [DBG] pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 12 KiB/s wr, 150 op/s 2024-07-28T19:35:06.276 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[23952]: cluster 2024-07-28T19:35:05.021622+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-07-28T19:35:06.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[23952]: audit 2024-07-28T19:35:05.097941+0000 mon.a (mon.0) 1268 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:06.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[23952]: audit 2024-07-28T19:35:05.106937+0000 mon.a (mon.0) 1269 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:06.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[23952]: audit 2024-07-28T19:35:05.108830+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:06.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[23952]: audit 2024-07-28T19:35:05.379462+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:06.277 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[23952]: audit 2024-07-28T19:35:05.399978+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.100:0/3125101805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:06.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[21828]: cluster 2024-07-28T19:35:04.997608+0000 mgr.y (mgr.15207) 34 : cluster [DBG] pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 12 KiB/s wr, 150 op/s 2024-07-28T19:35:06.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[21828]: cluster 2024-07-28T19:35:05.021622+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[21828]: audit 2024-07-28T19:35:05.097941+0000 mon.a (mon.0) 1268 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[21828]: audit 2024-07-28T19:35:05.106937+0000 mon.a (mon.0) 1269 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[21828]: audit 2024-07-28T19:35:05.108830+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[21828]: audit 2024-07-28T19:35:05.379462+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[21828]: audit 2024-07-28T19:35:05.399978+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.100:0/3125101805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[31715]: cluster 2024-07-28T19:35:04.997608+0000 mgr.y (mgr.15207) 34 : cluster [DBG] pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 12 KiB/s wr, 150 op/s 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[31715]: cluster 2024-07-28T19:35:05.021622+0000 mon.a (mon.0) 1267 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[31715]: audit 2024-07-28T19:35:05.097941+0000 mon.a (mon.0) 1268 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[31715]: audit 2024-07-28T19:35:05.106937+0000 mon.a (mon.0) 1269 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:06.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[31715]: audit 2024-07-28T19:35:05.108830+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:06.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[31715]: audit 2024-07-28T19:35:05.379462+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:06.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:06 smithi100 bash[31715]: audit 2024-07-28T19:35:05.399978+0000 mon.c (mon.2) 197 : audit [DBG] from='client.? 172.21.15.100:0/3125101805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:06.546 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:35:06.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[61012]: debug 2024-07-28T19:35:06.357+0000 7f6a7bd4d640 1 -- 172.21.15.138:0/555445157 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5642fa5c8d00 con 0x5642fa5bb400 2024-07-28T19:35:06.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:06 smithi138 bash[61012]: debug 2024-07-28T19:35:06.477+0000 7f6a7e5b6040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:35:06.749 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:35:07.378 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV2Parent (4165 ms) 2024-07-28T19:35:07.378 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-07-28T19:35:07.445 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:07.470 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:07.490 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:07.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:07 smithi100 bash[21828]: cluster 2024-07-28T19:35:06.111632+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-07-28T19:35:07.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:07 smithi100 bash[31715]: cluster 2024-07-28T19:35:06.111632+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-07-28T19:35:07.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:07 smithi138 bash[23952]: cluster 2024-07-28T19:35:06.111632+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-07-28T19:35:08.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:07 smithi138 bash[61012]: debug 2024-07-28T19:35:07.781+0000 7f6a7e5b6040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:35:08.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:07 smithi138 bash[61012]: debug 2024-07-28T19:35:07.849+0000 7f6a7e5b6040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:35:08.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:08 smithi138 bash[61012]: debug 2024-07-28T19:35:08.013+0000 7f6a7e5b6040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:35:08.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:08 smithi100 bash[21828]: cluster 2024-07-28T19:35:06.998392+0000 mgr.y (mgr.15207) 35 : cluster [DBG] pgmap v47: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 392 KiB/s rd, 23 KiB/s wr, 496 op/s 2024-07-28T19:35:08.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:08 smithi100 bash[21828]: cluster 2024-07-28T19:35:07.121831+0000 mon.a (mon.0) 1273 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-07-28T19:35:08.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:08 smithi100 bash[21828]: audit 2024-07-28T19:35:07.764945+0000 mon.a (mon.0) 1274 : audit [DBG] from='client.? 172.21.15.100:0/2004257046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:08.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:08 smithi100 bash[21828]: cluster 2024-07-28T19:35:07.913478+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-07-28T19:35:08.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:08 smithi100 bash[31715]: cluster 2024-07-28T19:35:06.998392+0000 mgr.y (mgr.15207) 35 : cluster [DBG] pgmap v47: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 392 KiB/s rd, 23 KiB/s wr, 496 op/s 2024-07-28T19:35:08.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:08 smithi100 bash[31715]: cluster 2024-07-28T19:35:07.121831+0000 mon.a (mon.0) 1273 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-07-28T19:35:08.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:08 smithi100 bash[31715]: audit 2024-07-28T19:35:07.764945+0000 mon.a (mon.0) 1274 : audit [DBG] from='client.? 172.21.15.100:0/2004257046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:08.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:08 smithi100 bash[31715]: cluster 2024-07-28T19:35:07.913478+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-07-28T19:35:08.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:08 smithi138 bash[23952]: cluster 2024-07-28T19:35:06.998392+0000 mgr.y (mgr.15207) 35 : cluster [DBG] pgmap v47: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 392 KiB/s rd, 23 KiB/s wr, 496 op/s 2024-07-28T19:35:08.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:08 smithi138 bash[23952]: cluster 2024-07-28T19:35:07.121831+0000 mon.a (mon.0) 1273 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-07-28T19:35:08.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:08 smithi138 bash[23952]: audit 2024-07-28T19:35:07.764945+0000 mon.a (mon.0) 1274 : audit [DBG] from='client.? 172.21.15.100:0/2004257046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:08.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:08 smithi138 bash[23952]: cluster 2024-07-28T19:35:07.913478+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-07-28T19:35:08.933 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:09.030 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:08 smithi138 bash[61012]: debug 2024-07-28T19:35:08.677+0000 7f6a7e5b6040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:35:09.030 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:08 smithi138 bash[61012]: debug 2024-07-28T19:35:08.749+0000 7f6a7e5b6040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:35:09.030 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:08 smithi138 bash[61012]: debug 2024-07-28T19:35:08.821+0000 7f6a7e5b6040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:35:09.070 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:09.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[61012]: debug 2024-07-28T19:35:09.029+0000 7f6a7e5b6040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:35:09.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[61012]: debug 2024-07-28T19:35:09.117+0000 7f6a7e5b6040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:35:09.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[61012]: debug 2024-07-28T19:35:09.197+0000 7f6a7e5b6040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:35:09.742 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[61012]: debug 2024-07-28T19:35:09.469+0000 7f6a7e5b6040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:35:09.742 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[61012]: debug 2024-07-28T19:35:09.541+0000 7f6a7e5b6040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:35:10.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[23952]: cluster 2024-07-28T19:35:08.916861+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-07-28T19:35:10.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[23952]: cluster 2024-07-28T19:35:08.999275+0000 mgr.y (mgr.15207) 36 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 395 KiB/s rd, 23 KiB/s wr, 499 op/s 2024-07-28T19:35:10.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[61012]: debug 2024-07-28T19:35:09.741+0000 7f6a7e5b6040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:35:10.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:09 smithi138 bash[61012]: debug 2024-07-28T19:35:09.821+0000 7f6a7e5b6040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:35:10.178 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:35:10.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:09 smithi100 bash[31715]: cluster 2024-07-28T19:35:08.916861+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-07-28T19:35:10.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:09 smithi100 bash[31715]: cluster 2024-07-28T19:35:08.999275+0000 mgr.y (mgr.15207) 36 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 395 KiB/s rd, 23 KiB/s wr, 499 op/s 2024-07-28T19:35:10.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:09 smithi100 bash[21828]: cluster 2024-07-28T19:35:08.916861+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-07-28T19:35:10.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:09 smithi100 bash[21828]: cluster 2024-07-28T19:35:08.999275+0000 mgr.y (mgr.15207) 36 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 395 KiB/s rd, 23 KiB/s wr, 499 op/s 2024-07-28T19:35:10.500 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:10 smithi138 bash[61012]: debug 2024-07-28T19:35:10.105+0000 7f6a7e5b6040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-28T19:35:10.500 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:10 smithi138 bash[61012]: debug 2024-07-28T19:35:10.313+0000 7f6a7e5b6040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:35:10.789 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:10 smithi138 bash[61012]: debug 2024-07-28T19:35:10.501+0000 7f6a7e5b6040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:35:10.789 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:10 smithi138 bash[61012]: debug 2024-07-28T19:35:10.585+0000 7f6a7e5b6040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:35:11.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:10 smithi138 bash[61012]: debug 2024-07-28T19:35:10.789+0000 7f6a7e5b6040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:35:11.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:10 smithi138 bash[23952]: cluster 2024-07-28T19:35:09.920667+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-07-28T19:35:11.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:10 smithi138 bash[23952]: audit 2024-07-28T19:35:10.147923+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.100:0/1017131450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV1ParentAbort (3776 ms) 2024-07-28T19:35:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-07-28T19:35:11.227 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:11.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:10 smithi100 bash[21828]: cluster 2024-07-28T19:35:09.920667+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-07-28T19:35:11.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:10 smithi100 bash[21828]: audit 2024-07-28T19:35:10.147923+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.100:0/1017131450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:11.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:10 smithi100 bash[31715]: cluster 2024-07-28T19:35:09.920667+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-07-28T19:35:11.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:10 smithi100 bash[31715]: audit 2024-07-28T19:35:10.147923+0000 mon.c (mon.2) 198 : audit [DBG] from='client.? 172.21.15.100:0/1017131450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:11.248 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:11.264 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:12.024 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:11 smithi138 bash[61012]: debug 2024-07-28T19:35:11.697+0000 7f6a7e5b6040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:35:12.024 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:11 smithi138 bash[61012]: debug 2024-07-28T19:35:11.781+0000 7f6a7e5b6040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:35:12.025 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:11 smithi138 bash[23952]: cluster 2024-07-28T19:35:10.926309+0000 mon.a (mon.0) 1278 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-07-28T19:35:12.025 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:11 smithi138 bash[23952]: audit 2024-07-28T19:35:10.951485+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.025 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:11 smithi138 bash[23952]: audit 2024-07-28T19:35:10.963973+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.025 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:11 smithi138 bash[23952]: cluster 2024-07-28T19:35:11.000100+0000 mgr.y (mgr.15207) 37 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:12.025 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:11 smithi138 bash[23952]: audit 2024-07-28T19:35:11.718419+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.025 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:11 smithi138 bash[23952]: audit 2024-07-28T19:35:11.728882+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[21828]: cluster 2024-07-28T19:35:10.926309+0000 mon.a (mon.0) 1278 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-07-28T19:35:12.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[21828]: audit 2024-07-28T19:35:10.951485+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[21828]: audit 2024-07-28T19:35:10.963973+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[21828]: cluster 2024-07-28T19:35:11.000100+0000 mgr.y (mgr.15207) 37 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:12.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[21828]: audit 2024-07-28T19:35:11.718419+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[21828]: audit 2024-07-28T19:35:11.728882+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[31715]: cluster 2024-07-28T19:35:10.926309+0000 mon.a (mon.0) 1278 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-07-28T19:35:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[31715]: audit 2024-07-28T19:35:10.951485+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[31715]: audit 2024-07-28T19:35:10.963973+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[31715]: cluster 2024-07-28T19:35:11.000100+0000 mgr.y (mgr.15207) 37 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[31715]: audit 2024-07-28T19:35:11.718419+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:11 smithi100 bash[31715]: audit 2024-07-28T19:35:11.728882+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:12.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: debug 2024-07-28T19:35:12.025+0000 7f6a7e5b6040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:35:12.304 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: debug 2024-07-28T19:35:12.101+0000 7f6a7e5b6040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:35:12.577 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: debug 2024-07-28T19:35:12.497+0000 7f6a7e5b6040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:35:12.577 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[60615]: ts=2024-07-28T19:35:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:12.985 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: debug 2024-07-28T19:35:12.577+0000 7f6a7e5b6040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:35:12.986 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: [28/Jul/2024:19:35:12] ENGINE Bus STARTING 2024-07-28T19:35:12.986 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: CherryPy Checker: 2024-07-28T19:35:12.986 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: The Application mounted at '' has an empty config. 2024-07-28T19:35:12.986 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: [28/Jul/2024:19:35:12] ENGINE Serving on http://:::9283 2024-07-28T19:35:12.986 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[61012]: [28/Jul/2024:19:35:12] ENGINE Bus STARTED 2024-07-28T19:35:13.023 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:13.171 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:13.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:35:12] "GET /metrics HTTP/1.1" 200 38816 "" "Prometheus/2.43.0" 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[21828]: cluster 2024-07-28T19:35:11.972357+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[21828]: audit 2024-07-28T19:35:12.509643+0000 mon.a (mon.0) 1284 : audit [DBG] from='client.? 172.21.15.100:0/420627144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[21828]: cluster 2024-07-28T19:35:12.605538+0000 mon.a (mon.0) 1285 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[21828]: cluster 2024-07-28T19:35:12.605655+0000 mon.a (mon.0) 1286 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[21828]: audit 2024-07-28T19:35:12.607040+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[21828]: audit 2024-07-28T19:35:12.607885+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[21828]: audit 2024-07-28T19:35:12.610150+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[21828]: audit 2024-07-28T19:35:12.611824+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[31715]: cluster 2024-07-28T19:35:11.972357+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-07-28T19:35:13.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[31715]: audit 2024-07-28T19:35:12.509643+0000 mon.a (mon.0) 1284 : audit [DBG] from='client.? 172.21.15.100:0/420627144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[31715]: cluster 2024-07-28T19:35:12.605538+0000 mon.a (mon.0) 1285 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:35:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[31715]: cluster 2024-07-28T19:35:12.605655+0000 mon.a (mon.0) 1286 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:35:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[31715]: audit 2024-07-28T19:35:12.607040+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:35:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[31715]: audit 2024-07-28T19:35:12.607885+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:35:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[31715]: audit 2024-07-28T19:35:12.610150+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:35:13.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:12 smithi100 bash[31715]: audit 2024-07-28T19:35:12.611824+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:35:13.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[23952]: cluster 2024-07-28T19:35:11.972357+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-07-28T19:35:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[23952]: audit 2024-07-28T19:35:12.509643+0000 mon.a (mon.0) 1284 : audit [DBG] from='client.? 172.21.15.100:0/420627144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[23952]: cluster 2024-07-28T19:35:12.605538+0000 mon.a (mon.0) 1285 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:35:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[23952]: cluster 2024-07-28T19:35:12.605655+0000 mon.a (mon.0) 1286 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:35:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[23952]: audit 2024-07-28T19:35:12.607040+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:35:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[23952]: audit 2024-07-28T19:35:12.607885+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:35:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[23952]: audit 2024-07-28T19:35:12.610150+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:35:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:12 smithi138 bash[23952]: audit 2024-07-28T19:35:12.611824+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.? 172.21.15.138:0/2658974118' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:35:13.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:13 smithi138 bash[60615]: ts=2024-07-28T19:35:13.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:14.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:13 smithi100 bash[21828]: cluster 2024-07-28T19:35:12.986365+0000 mon.a (mon.0) 1287 : cluster [DBG] mgrmap e33: y(active, since 38s), standbys: x 2024-07-28T19:35:14.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:14 smithi100 bash[21828]: cluster 2024-07-28T19:35:13.001100+0000 mgr.y (mgr.15207) 38 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 481 KiB/s rd, 26 KiB/s wr, 604 op/s 2024-07-28T19:35:14.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:14 smithi100 bash[21828]: cluster 2024-07-28T19:35:13.001564+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-07-28T19:35:14.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:14 smithi100 bash[31715]: cluster 2024-07-28T19:35:12.986365+0000 mon.a (mon.0) 1287 : cluster [DBG] mgrmap e33: y(active, since 38s), standbys: x 2024-07-28T19:35:14.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:14 smithi100 bash[31715]: cluster 2024-07-28T19:35:13.001100+0000 mgr.y (mgr.15207) 38 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 481 KiB/s rd, 26 KiB/s wr, 604 op/s 2024-07-28T19:35:14.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:14 smithi100 bash[31715]: cluster 2024-07-28T19:35:13.001564+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-07-28T19:35:14.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:13 smithi138 bash[23952]: cluster 2024-07-28T19:35:12.986365+0000 mon.a (mon.0) 1287 : cluster [DBG] mgrmap e33: y(active, since 38s), standbys: x 2024-07-28T19:35:14.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:13 smithi138 bash[23952]: cluster 2024-07-28T19:35:13.001100+0000 mgr.y (mgr.15207) 38 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 481 KiB/s rd, 26 KiB/s wr, 604 op/s 2024-07-28T19:35:14.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:13 smithi138 bash[23952]: cluster 2024-07-28T19:35:13.001564+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-07-28T19:35:14.381 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:35:14.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:14 smithi138 bash[60615]: ts=2024-07-28T19:35:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:14.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:14 smithi138 bash[60615]: ts=2024-07-28T19:35:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:15.270 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV2ParentAbort (4117 ms) 2024-07-28T19:35:15.271 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-07-28T19:35:15.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:15 smithi138 bash[23952]: cluster 2024-07-28T19:35:14.006392+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-07-28T19:35:15.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:15 smithi138 bash[23952]: audit 2024-07-28T19:35:14.877267+0000 mon.a (mon.0) 1290 : audit [DBG] from='client.? 172.21.15.100:0/699723517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:15.352 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:15.381 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:15.403 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:15.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:15 smithi100 bash[21828]: cluster 2024-07-28T19:35:14.006392+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-07-28T19:35:15.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:15 smithi100 bash[21828]: audit 2024-07-28T19:35:14.877267+0000 mon.a (mon.0) 1290 : audit [DBG] from='client.? 172.21.15.100:0/699723517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:15.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:15 smithi100 bash[31715]: cluster 2024-07-28T19:35:14.006392+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-07-28T19:35:15.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:15 smithi100 bash[31715]: audit 2024-07-28T19:35:14.877267+0000 mon.a (mon.0) 1290 : audit [DBG] from='client.? 172.21.15.100:0/699723517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:16.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:16 smithi138 bash[23952]: cluster 2024-07-28T19:35:15.001957+0000 mgr.y (mgr.15207) 39 : cluster [DBG] pgmap v59: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 472 KiB/s rd, 26 KiB/s wr, 592 op/s 2024-07-28T19:35:16.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:16 smithi138 bash[23952]: cluster 2024-07-28T19:35:15.008326+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-07-28T19:35:16.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:16 smithi100 bash[21828]: cluster 2024-07-28T19:35:15.001957+0000 mgr.y (mgr.15207) 39 : cluster [DBG] pgmap v59: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 472 KiB/s rd, 26 KiB/s wr, 592 op/s 2024-07-28T19:35:16.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:16 smithi100 bash[21828]: cluster 2024-07-28T19:35:15.008326+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-07-28T19:35:16.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:16 smithi100 bash[31715]: cluster 2024-07-28T19:35:15.001957+0000 mgr.y (mgr.15207) 39 : cluster [DBG] pgmap v59: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 472 KiB/s rd, 26 KiB/s wr, 592 op/s 2024-07-28T19:35:16.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:16 smithi100 bash[31715]: cluster 2024-07-28T19:35:15.008326+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-07-28T19:35:17.074 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:17.186 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:17.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:17 smithi138 bash[23952]: cluster 2024-07-28T19:35:16.028045+0000 mon.a (mon.0) 1292 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-07-28T19:35:17.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:17 smithi100 bash[21828]: cluster 2024-07-28T19:35:16.028045+0000 mon.a (mon.0) 1292 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-07-28T19:35:17.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:17 smithi100 bash[31715]: cluster 2024-07-28T19:35:16.028045+0000 mon.a (mon.0) 1292 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-07-28T19:35:18.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:18 smithi138 bash[23952]: cluster 2024-07-28T19:35:17.002914+0000 mgr.y (mgr.15207) 40 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 268 KiB/s rd, 26 KiB/s wr, 356 op/s 2024-07-28T19:35:18.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:18 smithi138 bash[23952]: cluster 2024-07-28T19:35:17.032403+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-07-28T19:35:18.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:18 smithi138 bash[23952]: audit 2024-07-28T19:35:17.269496+0000 mon.a (mon.0) 1294 : audit [DBG] from='client.? 172.21.15.100:0/276016003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:18.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:18 smithi138 bash[23952]: cluster 2024-07-28T19:35:17.916886+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-07-28T19:35:18.314 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:35:18.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:18 smithi100 bash[21828]: cluster 2024-07-28T19:35:17.002914+0000 mgr.y (mgr.15207) 40 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 268 KiB/s rd, 26 KiB/s wr, 356 op/s 2024-07-28T19:35:18.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:18 smithi100 bash[21828]: cluster 2024-07-28T19:35:17.032403+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-07-28T19:35:18.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:18 smithi100 bash[21828]: audit 2024-07-28T19:35:17.269496+0000 mon.a (mon.0) 1294 : audit [DBG] from='client.? 172.21.15.100:0/276016003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:18.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:18 smithi100 bash[21828]: cluster 2024-07-28T19:35:17.916886+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-07-28T19:35:18.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:18 smithi100 bash[31715]: cluster 2024-07-28T19:35:17.002914+0000 mgr.y (mgr.15207) 40 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 268 KiB/s rd, 26 KiB/s wr, 356 op/s 2024-07-28T19:35:18.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:18 smithi100 bash[31715]: cluster 2024-07-28T19:35:17.032403+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-07-28T19:35:18.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:18 smithi100 bash[31715]: audit 2024-07-28T19:35:17.269496+0000 mon.a (mon.0) 1294 : audit [DBG] from='client.? 172.21.15.100:0/276016003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:18.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:18 smithi100 bash[31715]: cluster 2024-07-28T19:35:17.916886+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-07-28T19:35:19.037 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:18 smithi138 bash[60615]: ts=2024-07-28T19:35:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:19.298 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (4028 ms) 2024-07-28T19:35:19.298 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-07-28T19:35:19.372 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:19.390 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:19.404 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:20.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:20 smithi138 bash[23952]: cluster 2024-07-28T19:35:19.003786+0000 mgr.y (mgr.15207) 41 : cluster [DBG] pgmap v65: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 26 KiB/s wr, 357 op/s 2024-07-28T19:35:20.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:20 smithi138 bash[23952]: cluster 2024-07-28T19:35:19.045809+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-07-28T19:35:20.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:20 smithi138 bash[23952]: audit 2024-07-28T19:35:19.639980+0000 mon.a (mon.0) 1297 : audit [DBG] from='client.? 172.21.15.100:0/1469705980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:20.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:20 smithi100 bash[31715]: cluster 2024-07-28T19:35:19.003786+0000 mgr.y (mgr.15207) 41 : cluster [DBG] pgmap v65: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 26 KiB/s wr, 357 op/s 2024-07-28T19:35:20.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:20 smithi100 bash[31715]: cluster 2024-07-28T19:35:19.045809+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-07-28T19:35:20.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:20 smithi100 bash[31715]: audit 2024-07-28T19:35:19.639980+0000 mon.a (mon.0) 1297 : audit [DBG] from='client.? 172.21.15.100:0/1469705980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:20.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:20 smithi100 bash[21828]: cluster 2024-07-28T19:35:19.003786+0000 mgr.y (mgr.15207) 41 : cluster [DBG] pgmap v65: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 26 KiB/s wr, 357 op/s 2024-07-28T19:35:20.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:20 smithi100 bash[21828]: cluster 2024-07-28T19:35:19.045809+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-07-28T19:35:20.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:20 smithi100 bash[21828]: audit 2024-07-28T19:35:19.639980+0000 mon.a (mon.0) 1297 : audit [DBG] from='client.? 172.21.15.100:0/1469705980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:21.078 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:21.207 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:21.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:21 smithi138 bash[23952]: cluster 2024-07-28T19:35:20.048795+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-07-28T19:35:21.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:21 smithi138 bash[23952]: audit 2024-07-28T19:35:20.380004+0000 mon.a (mon.0) 1299 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:21.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:21 smithi100 bash[21828]: cluster 2024-07-28T19:35:20.048795+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-07-28T19:35:21.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:21 smithi100 bash[21828]: audit 2024-07-28T19:35:20.380004+0000 mon.a (mon.0) 1299 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:21.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:21 smithi100 bash[31715]: cluster 2024-07-28T19:35:20.048795+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-07-28T19:35:21.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:21 smithi100 bash[31715]: audit 2024-07-28T19:35:20.380004+0000 mon.a (mon.0) 1299 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:22.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: cluster 2024-07-28T19:35:21.004599+0000 mgr.y (mgr.15207) 42 : cluster [DBG] pgmap v68: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:22.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: cluster 2024-07-28T19:35:21.067572+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-07-28T19:35:22.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.836368+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.846694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.848581+0000 mon.a (mon.0) 1303 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.850189+0000 mon.a (mon.0) 1304 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.859974+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.979724+0000 mon.a (mon.0) 1306 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.983238+0000 mon.a (mon.0) 1307 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.993291+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:21.994554+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:22.002933+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:22.004116+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:22.012398+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:22.014578+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:22.015736+0000 mon.a (mon.0) 1314 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-07-28T19:35:22.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[23952]: audit 2024-07-28T19:35:22.028152+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.100:0/1842398888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:22.376 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:35:22.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: cluster 2024-07-28T19:35:21.004599+0000 mgr.y (mgr.15207) 42 : cluster [DBG] pgmap v68: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: cluster 2024-07-28T19:35:21.067572+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.836368+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.846694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.848581+0000 mon.a (mon.0) 1303 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.850189+0000 mon.a (mon.0) 1304 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.859974+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.979724+0000 mon.a (mon.0) 1306 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.983238+0000 mon.a (mon.0) 1307 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:35:22.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.993291+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:21.994554+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:22.002933+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:22.004116+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:22.012398+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:22.014578+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:22.015736+0000 mon.a (mon.0) 1314 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[21828]: audit 2024-07-28T19:35:22.028152+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.100:0/1842398888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: cluster 2024-07-28T19:35:21.004599+0000 mgr.y (mgr.15207) 42 : cluster [DBG] pgmap v68: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:22.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: cluster 2024-07-28T19:35:21.067572+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.836368+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.846694+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.848581+0000 mon.a (mon.0) 1303 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.850189+0000 mon.a (mon.0) 1304 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.859974+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.979724+0000 mon.a (mon.0) 1306 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.983238+0000 mon.a (mon.0) 1307 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.993291+0000 mon.a (mon.0) 1308 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:21.994554+0000 mon.a (mon.0) 1309 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:22.002933+0000 mon.a (mon.0) 1310 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:22.004116+0000 mon.a (mon.0) 1311 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:22.012398+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-07-28T19:35:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:22.014578+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:35:22.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:22.015736+0000 mon.a (mon.0) 1314 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-07-28T19:35:22.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[31715]: audit 2024-07-28T19:35:22.028152+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.100:0/1842398888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:22.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:22 smithi138 bash[60615]: ts=2024-07-28T19:35:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:23.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:35:22] "GET /metrics HTTP/1.1" 200 38817 "" "Prometheus/2.43.0" 2024-07-28T19:35:23.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:23 smithi100 bash[21828]: cephadm 2024-07-28T19:35:21.984443+0000 mgr.y (mgr.15207) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-07-28T19:35:23.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:23 smithi100 bash[21828]: cephadm 2024-07-28T19:35:22.016701+0000 mgr.y (mgr.15207) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-07-28T19:35:23.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:23 smithi100 bash[21828]: cluster 2024-07-28T19:35:22.076109+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-07-28T19:35:23.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:23 smithi100 bash[31715]: cephadm 2024-07-28T19:35:21.984443+0000 mgr.y (mgr.15207) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-07-28T19:35:23.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:23 smithi100 bash[31715]: cephadm 2024-07-28T19:35:22.016701+0000 mgr.y (mgr.15207) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-07-28T19:35:23.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:23 smithi100 bash[31715]: cluster 2024-07-28T19:35:22.076109+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-07-28T19:35:23.346 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (4048 ms) 2024-07-28T19:35:23.347 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-07-28T19:35:23.430 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:23.460 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:23.482 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:23.505 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:23 smithi138 bash[23952]: cephadm 2024-07-28T19:35:21.984443+0000 mgr.y (mgr.15207) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-07-28T19:35:23.505 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:23 smithi138 bash[23952]: cephadm 2024-07-28T19:35:22.016701+0000 mgr.y (mgr.15207) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-07-28T19:35:23.505 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:23 smithi138 bash[23952]: cluster 2024-07-28T19:35:22.076109+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-07-28T19:35:23.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:23 smithi138 bash[60615]: ts=2024-07-28T19:35:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:24.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:24 smithi100 bash[31715]: cluster 2024-07-28T19:35:23.005408+0000 mgr.y (mgr.15207) 45 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 280 KiB/s rd, 17 KiB/s wr, 357 op/s 2024-07-28T19:35:24.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:24 smithi100 bash[31715]: cluster 2024-07-28T19:35:23.086556+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-07-28T19:35:24.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:24 smithi100 bash[21828]: cluster 2024-07-28T19:35:23.005408+0000 mgr.y (mgr.15207) 45 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 280 KiB/s rd, 17 KiB/s wr, 357 op/s 2024-07-28T19:35:24.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:24 smithi100 bash[21828]: cluster 2024-07-28T19:35:23.086556+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-07-28T19:35:24.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:24 smithi138 bash[23952]: cluster 2024-07-28T19:35:23.005408+0000 mgr.y (mgr.15207) 45 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 280 KiB/s rd, 17 KiB/s wr, 357 op/s 2024-07-28T19:35:24.450 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:24 smithi138 bash[23952]: cluster 2024-07-28T19:35:23.086556+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-07-28T19:35:24.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:24 smithi138 bash[60615]: ts=2024-07-28T19:35:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:24.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:24 smithi138 bash[60615]: ts=2024-07-28T19:35:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:25.111 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:25.224 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:25.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[21828]: cluster 2024-07-28T19:35:24.086316+0000 mon.a (mon.0) 1317 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-07-28T19:35:25.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[21828]: audit 2024-07-28T19:35:24.385419+0000 mon.a (mon.0) 1318 : audit [DBG] from='client.? 172.21.15.100:0/327598413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:25.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[21828]: audit 2024-07-28T19:35:24.776236+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:25.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[21828]: audit 2024-07-28T19:35:24.777981+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:35:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[21828]: audit 2024-07-28T19:35:24.779105+0000 mon.a (mon.0) 1321 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:35:25.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[21828]: audit 2024-07-28T19:35:24.780254+0000 mon.a (mon.0) 1322 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[31715]: cluster 2024-07-28T19:35:24.086316+0000 mon.a (mon.0) 1317 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-07-28T19:35:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[31715]: audit 2024-07-28T19:35:24.385419+0000 mon.a (mon.0) 1318 : audit [DBG] from='client.? 172.21.15.100:0/327598413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[31715]: audit 2024-07-28T19:35:24.776236+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[31715]: audit 2024-07-28T19:35:24.777981+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:35:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[31715]: audit 2024-07-28T19:35:24.779105+0000 mon.a (mon.0) 1321 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:35:25.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:25 smithi100 bash[31715]: audit 2024-07-28T19:35:24.780254+0000 mon.a (mon.0) 1322 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:25.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:25 smithi138 bash[23952]: cluster 2024-07-28T19:35:24.086316+0000 mon.a (mon.0) 1317 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-07-28T19:35:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:25 smithi138 bash[23952]: audit 2024-07-28T19:35:24.385419+0000 mon.a (mon.0) 1318 : audit [DBG] from='client.? 172.21.15.100:0/327598413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:25 smithi138 bash[23952]: audit 2024-07-28T19:35:24.776236+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' 2024-07-28T19:35:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:25 smithi138 bash[23952]: audit 2024-07-28T19:35:24.777981+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:35:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:25 smithi138 bash[23952]: audit 2024-07-28T19:35:24.779105+0000 mon.a (mon.0) 1321 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:35:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:25 smithi138 bash[23952]: audit 2024-07-28T19:35:24.780254+0000 mon.a (mon.0) 1322 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:26.351 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:26 smithi100 bash[21828]: cephadm 2024-07-28T19:35:24.770497+0000 mgr.y (mgr.15207) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-07-28T19:35:26.351 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:26 smithi100 bash[21828]: cephadm 2024-07-28T19:35:24.781528+0000 mgr.y (mgr.15207) 47 : cephadm [INF] Deploying daemon mon.a on smithi100 2024-07-28T19:35:26.351 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:26 smithi100 bash[21828]: cluster 2024-07-28T19:35:25.006180+0000 mgr.y (mgr.15207) 48 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 280 KiB/s rd, 17 KiB/s wr, 357 op/s 2024-07-28T19:35:26.351 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:26 smithi100 bash[21828]: cluster 2024-07-28T19:35:25.096783+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-07-28T19:35:26.351 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:26 smithi100 bash[31715]: cephadm 2024-07-28T19:35:24.770497+0000 mgr.y (mgr.15207) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-07-28T19:35:26.352 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:26 smithi100 bash[31715]: cephadm 2024-07-28T19:35:24.781528+0000 mgr.y (mgr.15207) 47 : cephadm [INF] Deploying daemon mon.a on smithi100 2024-07-28T19:35:26.352 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:26 smithi100 bash[31715]: cluster 2024-07-28T19:35:25.006180+0000 mgr.y (mgr.15207) 48 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 280 KiB/s rd, 17 KiB/s wr, 357 op/s 2024-07-28T19:35:26.352 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:26 smithi100 bash[31715]: cluster 2024-07-28T19:35:25.096783+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-07-28T19:35:26.465 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:35:26.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:26 smithi138 bash[23952]: cephadm 2024-07-28T19:35:24.770497+0000 mgr.y (mgr.15207) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-07-28T19:35:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:26 smithi138 bash[23952]: cephadm 2024-07-28T19:35:24.781528+0000 mgr.y (mgr.15207) 47 : cephadm [INF] Deploying daemon mon.a on smithi100 2024-07-28T19:35:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:26 smithi138 bash[23952]: cluster 2024-07-28T19:35:25.006180+0000 mgr.y (mgr.15207) 48 : cluster [DBG] pgmap v74: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 280 KiB/s rd, 17 KiB/s wr, 357 op/s 2024-07-28T19:35:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:26 smithi138 bash[23952]: cluster 2024-07-28T19:35:25.096783+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-07-28T19:35:27.299 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.299 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.300 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[21828]: cluster 2024-07-28T19:35:26.117747+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-07-28T19:35:27.300 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[21828]: audit 2024-07-28T19:35:26.745088+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.100:0/3697627249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:27.300 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.300 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.301 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.301 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.301 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.302 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.302 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[31715]: cluster 2024-07-28T19:35:26.117747+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-07-28T19:35:27.302 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[31715]: audit 2024-07-28T19:35:26.745088+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.100:0/3697627249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:27.302 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:27.321 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (3975 ms) 2024-07-28T19:35:27.321 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-07-28T19:35:27.375 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:27.396 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:27.413 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:27.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:27 smithi138 bash[23952]: cluster 2024-07-28T19:35:26.117747+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-07-28T19:35:27.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:27 smithi138 bash[23952]: audit 2024-07-28T19:35:26.745088+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.100:0/3697627249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:27.584 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[73305]: [28/Jul/2024:19:35:27] ENGINE Bus STOPPING 2024-07-28T19:35:27.584 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:27 smithi100 systemd[1]: Stopping Ceph mon.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:35:27.584 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[21828]: debug 2024-07-28T19:35:27.406+0000 7f9aff55e640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-28T19:35:27.585 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[21828]: debug 2024-07-28T19:35:27.406+0000 7f9aff55e640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-07-28T19:35:28.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:28 smithi100 bash[80072]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-a 2024-07-28T19:35:28.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[73305]: [28/Jul/2024:19:35:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:35:28.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[73305]: [28/Jul/2024:19:35:27] ENGINE Bus STOPPED 2024-07-28T19:35:28.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[73305]: [28/Jul/2024:19:35:27] ENGINE Bus STARTING 2024-07-28T19:35:28.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[73305]: [28/Jul/2024:19:35:27] ENGINE Serving on http://:::9283 2024-07-28T19:35:28.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:27 smithi100 bash[73305]: [28/Jul/2024:19:35:27] ENGINE Bus STARTED 2024-07-28T19:35:28.678 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.679 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.679 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.679 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.680 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.680 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.680 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.681 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.681 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:28 smithi100 bash[80144]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-a 2024-07-28T19:35:28.681 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a.service: Deactivated successfully. 2024-07-28T19:35:28.681 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: Stopped Ceph mon.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:35:28.681 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:35:28.681 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:28 smithi100 systemd[1]: Started Ceph mon.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:35:29.054 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:28 smithi138 bash[60615]: ts=2024-07-28T19:35:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:29.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.730+0000 7f8fcb736b00 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-28T19:35:29.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.730+0000 7f8fcb736b00 0 ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable), process ceph-mon, pid 7 2024-07-28T19:35:29.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.730+0000 7f8fcb736b00 0 pidfile_write: ignore empty --pid-file 2024-07-28T19:35:29.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 0 load: jerasure load: lrc 2024-07-28T19:35:29.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: RocksDB version: 7.9.2 2024-07-28T19:35:29.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Git sha 0 2024-07-28T19:35:29.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Compile date 2024-07-22 20:55:56 2024-07-28T19:35:29.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: DB SUMMARY 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: DB Session ID: HIAO9N0WGWOTVN2S0LC6 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: CURRENT file: CURRENT 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: IDENTITY file: IDENTITY 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 1726 Bytes 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000042.sst 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000040.log size: 1892267 ; 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.error_if_exists: 0 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.create_if_missing: 0 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.paranoid_checks: 1 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.env: 0x55f19afeebc0 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.fs: PosixFileSystem 2024-07-28T19:35:29.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.info_log: 0x55f19c138be0 2024-07-28T19:35:29.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-28T19:35:29.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.statistics: (nil) 2024-07-28T19:35:29.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.use_fsync: 0 2024-07-28T19:35:29.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_log_file_size: 0 2024-07-28T19:35:29.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-28T19:35:29.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.allow_fallocate: 1 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.use_direct_reads: 0 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.create_missing_column_families: 0 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.db_log_dir: 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.wal_dir: 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-28T19:35:29.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.advise_random_on_open: 1 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.write_buffer_manager: 0x55f19c146b40 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-28T19:35:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.rate_limiter: (nil) 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.unordered_write: 0 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.row_cache: None 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.wal_filter: None 2024-07-28T19:35:29.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.two_write_queues: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.manual_wal_flush: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.wal_compression: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.atomic_flush: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.log_readahead_size: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-28T19:35:30.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_background_jobs: 2 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_background_compactions: -1 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_subcompactions: 1 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_total_wal_size: 0 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-28T19:35:30.001 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_open_files: -1 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bytes_per_sync: 0 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_background_flushes: -1 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Compression algorithms supported: 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: kZSTD supported: 0 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: kXpressCompression supported: 0 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: kBZip2Compression supported: 0 2024-07-28T19:35:30.002 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-28T19:35:30.003 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: kLZ4Compression supported: 1 2024-07-28T19:35:30.003 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: kZlibCompression supported: 1 2024-07-28T19:35:30.003 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-28T19:35:30.003 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: kSnappyCompression supported: 1 2024-07-28T19:35:30.003 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-28T19:35:30.003 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-07-28T19:35:30.003 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-07-28T19:35:30.004 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-07-28T19:35:30.004 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.merge_operator: 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_filter: None 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_filter_factory: None 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f19c138820) 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: cache_index_and_filter_blocks: 1 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: pin_top_level_index_and_filter: 1 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: index_type: 0 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: data_block_index_type: 0 2024-07-28T19:35:30.005 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: index_shortening: 1 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: data_block_hash_table_util_ratio: 0.750000 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: checksum: 4 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: no_block_cache: 0 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: block_cache: 0x55f19c12d350 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: block_cache_name: BinnedLRUCache 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: block_cache_options: 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: capacity : 536870912 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: num_shard_bits : 4 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: strict_capacity_limit : 0 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: high_pri_pool_ratio: 0.000 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: block_cache_compressed: (nil) 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: persistent_cache: (nil) 2024-07-28T19:35:30.006 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: block_size: 4096 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: block_size_deviation: 10 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: block_restart_interval: 16 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: index_block_restart_interval: 1 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: metadata_block_size: 4096 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: partition_filters: 0 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: use_delta_encoding: 1 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: filter_policy: bloomfilter 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: whole_key_filtering: 1 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: verify_compression: 0 2024-07-28T19:35:30.007 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: read_amp_bytes_per_bit: 0 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: format_version: 5 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: enable_index_compression: 1 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: block_align: 0 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: max_auto_readahead_size: 262144 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: prepopulate_block_cache: 0 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: initial_auto_readahead_size: 8192 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: num_file_reads_for_auto_readahead: 2 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression: NoCompression 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-28T19:35:30.008 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.num_levels: 7 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-28T19:35:30.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.level: 32767 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.enabled: false 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-28T19:35:30.010 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-28T19:35:30.011 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.arena_block_size: 1048576 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-28T19:35:30.012 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.table_properties_collectors: 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.inplace_update_support: 0 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.bloom_locality: 0 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.max_successive_merges: 0 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-28T19:35:30.013 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.force_consistency_checks: 1 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.ttl: 2592000 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.enable_blob_files: false 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.min_blob_size: 0 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.blob_file_size: 268435456 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-28T19:35:30.014 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.blob_file_starting_level: 0 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fc850f640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 42.sst 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 44, last_sequence is 18319, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f5893a4e-1044-4197-a091-cfdc818067c7 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195329739734, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.734+0000 7f8fcb736b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-07-28T19:35:30.015 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.742+0000 7f8fcb736b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195329746964, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 1793911, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18307, "largest_seqno": 19437, "table_properties": {"data_size": 1787517, "index_size": 4099, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1285, "raw_key_size": 11756, "raw_average_key_size": 23, "raw_value_size": 1776072, "raw_average_value_size": 3588, "num_data_blocks": 180, "num_entries": 495, "num_filter_entries": 495, "num_deletions": 5, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1722195329, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f5893a4e-1044-4197-a091-cfdc818067c7", "db_session_id": "HIAO9N0WGWOTVN2S0LC6", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.742+0000 7f8fcb736b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195329747098, "job": 1, "event": "recovery_finished"} 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.742+0000 7f8fcb736b00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.742+0000 7f8fcb736b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f19c158e00 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 4 rocksdb: DB pointer 0x55f19c250000 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fc1501640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fc1501640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: ** DB Stats ** 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-28T19:35:30.016 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: ** Compaction Stats [default] ** 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: L0 1/0 1.71 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 446.3 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: L6 1/0 9.51 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Sum 2/0 11.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 446.3 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 446.3 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: ** Compaction Stats [default] ** 2024-07-28T19:35:30.017 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 446.3 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Flush(GB): cumulative 0.002, interval 0.002 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Cumulative compaction: 0.00 GB write, 145.09 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Interval compaction: 0.00 GB write, 145.09 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:35:30.018 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Block cache BinnedLRUCache@0x55f19c12d350#7 capacity: 512.00 MB usage: 5.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2e-05 secs_since: 0 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 0 starting mon.a rank 0 at public addrs [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] at bind addrs [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 1 mon.a@-1(???) e3 preinit fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 0 mon.a@-1(???).mds e10 new map 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 0 mon.a@-1(???).mds e10 print_map 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: e10 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: legacy client fscid: 1 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Filesystem 'foo' (1) 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: fs_name foo 2024-07-28T19:35:30.019 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: epoch 10 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: created 2024-07-28T19:29:15.255828+0000 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: modified 2024-07-28T19:29:25.471713+0000 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: tableserver 0 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: root 0 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: session_timeout 60 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: session_autoclose 300 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: max_file_size 1099511627776 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: max_xattr_size 65536 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: required_client_features {} 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: last_failure 0 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: last_failure_osd_epoch 70 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:35:30.020 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: max_mds 1 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: in 0 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: up {0=24331} 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: failed 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: damaged 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: stopped 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: data_pools [4] 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: metadata_pool 3 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: inline_data disabled 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: balancer 2024-07-28T19:35:30.021 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: bal_rank_mask -1 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: standby_count_wanted 1 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: [mds.foo.smithi138.nphkis{0:24331} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] compat {c=[1],r=[1],i=[7ff]}] 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: Standby daemons: 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: [mds.foo.smithi100.ppcqpk{-1:14607} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.100:6834/1539587085,v1:172.21.15.100:6835/1539587085] compat {c=[1],r=[1],i=[7ff]}] 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 0 mon.a@-1(???).osd e295 crush map has features 3314933000854323200, adjusting msgr requires 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 0 mon.a@-1(???).osd e295 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 0 mon.a@-1(???).osd e295 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 0 mon.a@-1(???).osd e295 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:35:30.022 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:29 smithi100 bash[80199]: debug 2024-07-28T19:35:29.746+0000 7f8fcb736b00 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-07-28T19:35:31.254 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: debug 2024-07-28T19:35:31.250+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:35:31.254 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: audit 2024-07-28T19:35:27.413220+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:35:31.254 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: audit 2024-07-28T19:35:27.414702+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:35:31.254 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: audit 2024-07-28T19:35:27.415391+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:35:31.254 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:29.007781+0000 mgr.y (mgr.15207) 50 : cluster [DBG] pgmap v79: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 17 KiB/s wr, 364 op/s 2024-07-28T19:35:31.254 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: audit 2024-07-28T19:35:29.092212+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.100:0/1273477184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:31.254 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.172297+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.194117+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.200191+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.200716+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.200739+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.201875+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e33: y(active, since 56s), standbys: x 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.202292+0000 mon.a (mon.0) 7 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.202307+0000 mon.a (mon.0) 8 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.202314+0000 mon.a (mon.0) 9 : cluster [WRN] application not enabled on pool 'test-librbd-smithi100-68802-8' 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.202321+0000 mon.a (mon.0) 10 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.235957+0000 mon.a (mon.0) 11 : cluster [DBG] mgrmap e34: y(active, since 56s), standbys: x 2024-07-28T19:35:31.255 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[80199]: audit 2024-07-28T19:35:31.236050+0000 mon.a (mon.0) 12 : audit [INF] from='mgr.15207 ' entity='' 2024-07-28T19:35:31.499 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable)": 1 2024-07-28T19:35:31.500 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[73305]: ignoring --setuser ceph since I am not root 2024-07-28T19:35:31.500 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[73305]: ignoring --setgroup ceph since I am not root 2024-07-28T19:35:31.500 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[73305]: debug 2024-07-28T19:35:31.298+0000 7f96d4eba640 1 -- 172.21.15.100:0/1221444028 <== mon.0 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a37865ed00 con 0x55a378651400 2024-07-28T19:35:31.500 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[73305]: debug 2024-07-28T19:35:31.414+0000 7f96d7715040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:35:31.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: audit 2024-07-28T19:35:27.413220+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:35:31.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: audit 2024-07-28T19:35:27.414702+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:35:31.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: audit 2024-07-28T19:35:27.415391+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:35:31.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:29.007781+0000 mgr.y (mgr.15207) 50 : cluster [DBG] pgmap v79: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 17 KiB/s wr, 364 op/s 2024-07-28T19:35:31.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: audit 2024-07-28T19:35:29.092212+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.100:0/1273477184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:31.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.172297+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-07-28T19:35:31.501 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.194117+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:35:31.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.200191+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:35:31.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.200716+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:35:31.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.200739+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-07-28T19:35:31.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.201875+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e33: y(active, since 56s), standbys: x 2024-07-28T19:35:31.502 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.202292+0000 mon.a (mon.0) 7 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-07-28T19:35:31.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.202307+0000 mon.a (mon.0) 8 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-07-28T19:35:31.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.202314+0000 mon.a (mon.0) 9 : cluster [WRN] application not enabled on pool 'test-librbd-smithi100-68802-8' 2024-07-28T19:35:31.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.202321+0000 mon.a (mon.0) 10 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-07-28T19:35:31.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.235957+0000 mon.a (mon.0) 11 : cluster [DBG] mgrmap e34: y(active, since 56s), standbys: x 2024-07-28T19:35:31.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:31 smithi100 bash[31715]: audit 2024-07-28T19:35:31.236050+0000 mon.a (mon.0) 12 : audit [INF] from='mgr.15207 ' entity='' 2024-07-28T19:35:31.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[61012]: ignoring --setuser ceph since I am not root 2024-07-28T19:35:31.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[61012]: ignoring --setgroup ceph since I am not root 2024-07-28T19:35:31.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[61012]: debug 2024-07-28T19:35:31.292+0000 7f3ab347c640 1 -- 172.21.15.138:0/3210323999 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55808bd3ed00 con 0x55808bd31400 2024-07-28T19:35:31.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[61012]: debug 2024-07-28T19:35:31.408+0000 7f3ab5cd7040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-07-28T19:35:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: audit 2024-07-28T19:35:27.413220+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:35:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: audit 2024-07-28T19:35:27.414702+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:35:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: audit 2024-07-28T19:35:27.415391+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15207 172.21.15.100:0/4207211565' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:35:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:29.007781+0000 mgr.y (mgr.15207) 50 : cluster [DBG] pgmap v79: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 17 KiB/s wr, 364 op/s 2024-07-28T19:35:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: audit 2024-07-28T19:35:29.092212+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.100:0/1273477184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.172297+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.194117+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.200191+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.200716+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.200739+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.201875+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e33: y(active, since 56s), standbys: x 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.202292+0000 mon.a (mon.0) 7 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.202307+0000 mon.a (mon.0) 8 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.202314+0000 mon.a (mon.0) 9 : cluster [WRN] application not enabled on pool 'test-librbd-smithi100-68802-8' 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.202321+0000 mon.a (mon.0) 10 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-07-28T19:35:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.235957+0000 mon.a (mon.0) 11 : cluster [DBG] mgrmap e34: y(active, since 56s), standbys: x 2024-07-28T19:35:31.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:31 smithi138 bash[23952]: audit 2024-07-28T19:35:31.236050+0000 mon.a (mon.0) 12 : audit [INF] from='mgr.15207 ' entity='' 2024-07-28T19:35:32.284 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:32.382 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch ps' 2024-07-28T19:35:32.390 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:32.554 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:32 smithi138 bash[60615]: ts=2024-07-28T19:35:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:32.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:32 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.253888+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:35:32.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:32 smithi138 bash[23952]: cluster 2024-07-28T19:35:31.266340+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-07-28T19:35:32.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:32 smithi138 bash[23952]: audit 2024-07-28T19:35:31.494625+0000 mon.a (mon.0) 15 : audit [DBG] from='client.? 172.21.15.100:0/352681457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:32.698 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.253888+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:35:32.698 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[31715]: cluster 2024-07-28T19:35:31.266340+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-07-28T19:35:32.698 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[31715]: audit 2024-07-28T19:35:31.494625+0000 mon.a (mon.0) 15 : audit [DBG] from='client.? 172.21.15.100:0/352681457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:32.698 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.253888+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:35:32.699 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[80199]: cluster 2024-07-28T19:35:31.266340+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-07-28T19:35:32.699 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[80199]: audit 2024-07-28T19:35:31.494625+0000 mon.a (mon.0) 15 : audit [DBG] from='client.? 172.21.15.100:0/352681457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:35:32.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[73305]: debug 2024-07-28T19:35:32.702+0000 7f96d7715040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:35:32.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[73305]: debug 2024-07-28T19:35:32.766+0000 7f96d7715040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:35:32.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:32 smithi100 bash[73305]: debug 2024-07-28T19:35:32.922+0000 7f96d7715040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:35:33.054 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:32 smithi138 bash[61012]: debug 2024-07-28T19:35:32.640+0000 7f3ab5cd7040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-07-28T19:35:33.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:32 smithi138 bash[61012]: debug 2024-07-28T19:35:32.708+0000 7f3ab5cd7040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-07-28T19:35:33.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:32 smithi138 bash[61012]: debug 2024-07-28T19:35:32.860+0000 7f3ab5cd7040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-07-28T19:35:33.544 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:33 smithi138 bash[23952]: cluster 2024-07-28T19:35:32.271917+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-07-28T19:35:33.544 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:33 smithi138 bash[60615]: ts=2024-07-28T19:35:33.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:33.622 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:33 smithi100 bash[80199]: cluster 2024-07-28T19:35:32.271917+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-07-28T19:35:33.623 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:33 smithi100 bash[31715]: cluster 2024-07-28T19:35:32.271917+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-07-28T19:35:33.627 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:35:33.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:33 smithi138 bash[61012]: debug 2024-07-28T19:35:33.544+0000 7f3ab5cd7040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:35:33.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:33 smithi138 bash[61012]: debug 2024-07-28T19:35:33.616+0000 7f3ab5cd7040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:35:33.805 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:33 smithi138 bash[61012]: debug 2024-07-28T19:35:33.696+0000 7f3ab5cd7040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:35:33.973 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:33 smithi100 bash[73305]: debug 2024-07-28T19:35:33.626+0000 7f96d7715040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-07-28T19:35:33.973 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:33 smithi100 bash[73305]: debug 2024-07-28T19:35:33.706+0000 7f96d7715040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-07-28T19:35:33.973 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:33 smithi100 bash[73305]: debug 2024-07-28T19:35:33.786+0000 7f96d7715040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-07-28T19:35:34.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:33 smithi100 bash[73305]: debug 2024-07-28T19:35:33.974+0000 7f96d7715040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:35:34.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:34 smithi100 bash[73305]: debug 2024-07-28T19:35:34.054+0000 7f96d7715040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:35:34.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:34 smithi100 bash[73305]: debug 2024-07-28T19:35:34.122+0000 7f96d7715040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:35:34.278 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:33 smithi138 bash[61012]: debug 2024-07-28T19:35:33.892+0000 7f3ab5cd7040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-07-28T19:35:34.278 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:33 smithi138 bash[61012]: debug 2024-07-28T19:35:33.972+0000 7f3ab5cd7040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-07-28T19:35:34.278 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:34 smithi138 bash[61012]: debug 2024-07-28T19:35:34.040+0000 7f3ab5cd7040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-07-28T19:35:34.523 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (7201 ms) 2024-07-28T19:35:34.523 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-07-28T19:35:34.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:34 smithi138 bash[23952]: cluster 2024-07-28T19:35:33.277515+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-07-28T19:35:34.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:34 smithi138 bash[61012]: debug 2024-07-28T19:35:34.308+0000 7f3ab5cd7040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:35:34.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:34 smithi138 bash[61012]: debug 2024-07-28T19:35:34.388+0000 7f3ab5cd7040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:35:34.595 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:34.622 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:34.636 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:34.711 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:34 smithi100 bash[80199]: cluster 2024-07-28T19:35:33.277515+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-07-28T19:35:34.711 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:34 smithi100 bash[31715]: cluster 2024-07-28T19:35:33.277515+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-07-28T19:35:34.711 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:34 smithi100 bash[73305]: debug 2024-07-28T19:35:34.406+0000 7f96d7715040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-07-28T19:35:34.711 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:34 smithi100 bash[73305]: debug 2024-07-28T19:35:34.490+0000 7f96d7715040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-07-28T19:35:34.937 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:34 smithi138 bash[61012]: debug 2024-07-28T19:35:34.592+0000 7f3ab5cd7040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:35:34.937 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:34 smithi138 bash[61012]: debug 2024-07-28T19:35:34.676+0000 7f3ab5cd7040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:35:34.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:34 smithi100 bash[73305]: debug 2024-07-28T19:35:34.714+0000 7f96d7715040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-07-28T19:35:34.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:34 smithi100 bash[73305]: debug 2024-07-28T19:35:34.798+0000 7f96d7715040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-07-28T19:35:35.282 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:34 smithi138 bash[61012]: debug 2024-07-28T19:35:34.936+0000 7f3ab5cd7040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-28T19:35:35.283 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:35 smithi138 bash[61012]: debug 2024-07-28T19:35:35.116+0000 7f3ab5cd7040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:35:35.405 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[80199]: cluster 2024-07-28T19:35:34.299861+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-07-28T19:35:35.405 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[80199]: cluster 2024-07-28T19:35:34.765684+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-07-28T19:35:35.405 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[31715]: cluster 2024-07-28T19:35:34.299861+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-07-28T19:35:35.405 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[31715]: cluster 2024-07-28T19:35:34.765684+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-07-28T19:35:35.406 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[73305]: debug 2024-07-28T19:35:35.058+0000 7f96d7715040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-07-28T19:35:35.406 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[73305]: debug 2024-07-28T19:35:35.238+0000 7f96d7715040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:35:35.544 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:35 smithi138 bash[23952]: cluster 2024-07-28T19:35:34.299861+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-07-28T19:35:35.544 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:35 smithi138 bash[23952]: cluster 2024-07-28T19:35:34.765684+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-07-28T19:35:35.544 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:35 smithi138 bash[61012]: debug 2024-07-28T19:35:35.280+0000 7f3ab5cd7040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:35:35.544 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:35 smithi138 bash[61012]: debug 2024-07-28T19:35:35.364+0000 7f3ab5cd7040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:35:35.668 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[73305]: debug 2024-07-28T19:35:35.406+0000 7f96d7715040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-07-28T19:35:35.668 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[73305]: debug 2024-07-28T19:35:35.486+0000 7f96d7715040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-07-28T19:35:35.668 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:35 smithi100 bash[73305]: debug 2024-07-28T19:35:35.670+0000 7f96d7715040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:35:35.783 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:35.804 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:35 smithi138 bash[61012]: debug 2024-07-28T19:35:35.544+0000 7f3ab5cd7040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-07-28T19:35:35.898 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:36.707 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:36 smithi138 bash[61012]: debug 2024-07-28T19:35:36.400+0000 7f3ab5cd7040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:35:36.707 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:36 smithi138 bash[61012]: debug 2024-07-28T19:35:36.476+0000 7f3ab5cd7040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:35:36.871 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:36 smithi100 bash[80199]: cluster 2024-07-28T19:35:35.768322+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-07-28T19:35:36.872 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:36 smithi100 bash[73305]: debug 2024-07-28T19:35:36.550+0000 7f96d7715040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-07-28T19:35:36.872 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:36 smithi100 bash[73305]: debug 2024-07-28T19:35:36.626+0000 7f96d7715040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-07-28T19:35:36.872 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:36 smithi100 bash[31715]: cluster 2024-07-28T19:35:35.768322+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-07-28T19:35:37.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:36 smithi138 bash[23952]: cluster 2024-07-28T19:35:35.768322+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-07-28T19:35:37.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:36 smithi138 bash[61012]: debug 2024-07-28T19:35:36.704+0000 7f3ab5cd7040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:35:37.055 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:36 smithi138 bash[61012]: debug 2024-07-28T19:35:36.788+0000 7f3ab5cd7040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:35:37.102 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:35:37.168 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:35:37.242 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:36 smithi100 bash[73305]: debug 2024-07-28T19:35:36.874+0000 7f96d7715040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-07-28T19:35:37.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:36 smithi100 bash[73305]: debug 2024-07-28T19:35:36.966+0000 7f96d7715040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-07-28T19:35:37.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[61012]: debug 2024-07-28T19:35:37.216+0000 7f3ab5cd7040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:35:37.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[61012]: debug 2024-07-28T19:35:37.296+0000 7f3ab5cd7040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:35:37.554 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[61012]: [28/Jul/2024:19:35:37] ENGINE Bus STARTING 2024-07-28T19:35:37.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[61012]: CherryPy Checker: 2024-07-28T19:35:37.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[61012]: The Application mounted at '' has an empty config. 2024-07-28T19:35:37.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[61012]: [28/Jul/2024:19:35:37] ENGINE Serving on http://:::9283 2024-07-28T19:35:37.555 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[61012]: [28/Jul/2024:19:35:37] ENGINE Bus STARTED 2024-07-28T19:35:37.599 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.410+0000 7f96d7715040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-07-28T19:35:37.599 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.490+0000 7f96d7715040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-07-28T19:35:37.673 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (3151 ms) 2024-07-28T19:35:37.673 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-07-28T19:35:37.725 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:37.746 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 0~10 2024-07-28T19:35:37.761 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:37.896 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: cluster 2024-07-28T19:35:36.774163+0000 mon.a (mon.0) 21 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.325601+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: cluster 2024-07-28T19:35:37.326051+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: cluster 2024-07-28T19:35:37.326196+0000 mon.a (mon.0) 23 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.326521+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.328090+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.328543+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: cluster 2024-07-28T19:35:37.499726+0000 mon.a (mon.0) 24 : cluster [INF] Active manager daemon y restarted 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: cluster 2024-07-28T19:35:37.501172+0000 mon.a (mon.0) 25 : cluster [INF] Activating manager daemon y 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: cluster 2024-07-28T19:35:37.514963+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: cluster 2024-07-28T19:35:37.517941+0000 mon.a (mon.0) 27 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0170793s), standbys: x 2024-07-28T19:35:37.897 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.524786+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.525911+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.526710+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.527341+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.528107+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.528833+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.529518+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.529871+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.530543+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.530844+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.531123+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.531431+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.531755+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.532009+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:35:37.898 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.532392+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.532788+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.533097+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.533693+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: cluster 2024-07-28T19:35:37.541122+0000 mon.a (mon.0) 46 : cluster [INF] Manager daemon y is now available 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.566179+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.611118+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.615558+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.620011+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[80199]: audit 2024-07-28T19:35:37.645155+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:35:37.899 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: [28/Jul/2024:19:35:37] ENGINE Bus STARTING 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: CherryPy Checker: 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: The Application mounted at '' has an empty config. 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.686+0000 7f9642cca640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.686+0000 7f9642cca640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.686+0000 7f9642cca640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.686+0000 7f9642cca640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.686+0000 7f9642cca640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.694+0000 7f963e441640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.694+0000 7f963e441640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.694+0000 7f963e441640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.694+0000 7f963e441640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.900 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: debug 2024-07-28T19:35:37.694+0000 7f963e441640 -1 client.0 error registering admin socket command: (17) File exists 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: [28/Jul/2024:19:35:37] ENGINE Serving on http://:::9283 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[73305]: [28/Jul/2024:19:35:37] ENGINE Bus STARTED 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: cluster 2024-07-28T19:35:36.774163+0000 mon.a (mon.0) 21 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.325601+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: cluster 2024-07-28T19:35:37.326051+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: cluster 2024-07-28T19:35:37.326196+0000 mon.a (mon.0) 23 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.326521+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.328090+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:35:37.901 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.328543+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: cluster 2024-07-28T19:35:37.499726+0000 mon.a (mon.0) 24 : cluster [INF] Active manager daemon y restarted 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: cluster 2024-07-28T19:35:37.501172+0000 mon.a (mon.0) 25 : cluster [INF] Activating manager daemon y 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: cluster 2024-07-28T19:35:37.514963+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: cluster 2024-07-28T19:35:37.517941+0000 mon.a (mon.0) 27 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0170793s), standbys: x 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.524786+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.525911+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.526710+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.527341+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.528107+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.528833+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.529518+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:35:37.902 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.529871+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.530543+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.530844+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.531123+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.531431+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.531755+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.532009+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.532392+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.532788+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.533097+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.533693+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: cluster 2024-07-28T19:35:37.541122+0000 mon.a (mon.0) 46 : cluster [INF] Manager daemon y is now available 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.566179+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.611118+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:37.903 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.615558+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:37.904 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.620011+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:35:37.904 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:37 smithi100 bash[31715]: audit 2024-07-28T19:35:37.645155+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:35:38.041 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: cluster 2024-07-28T19:35:36.774163+0000 mon.a (mon.0) 21 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-07-28T19:35:38.041 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.325601+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-07-28T19:35:38.041 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: cluster 2024-07-28T19:35:37.326051+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x restarted 2024-07-28T19:35:38.041 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: cluster 2024-07-28T19:35:37.326196+0000 mon.a (mon.0) 23 : cluster [DBG] Standby manager daemon x started 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.326521+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.328090+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.328543+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.138:0/602933302' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: cluster 2024-07-28T19:35:37.499726+0000 mon.a (mon.0) 24 : cluster [INF] Active manager daemon y restarted 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: cluster 2024-07-28T19:35:37.501172+0000 mon.a (mon.0) 25 : cluster [INF] Activating manager daemon y 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: cluster 2024-07-28T19:35:37.514963+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: cluster 2024-07-28T19:35:37.517941+0000 mon.a (mon.0) 27 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0170793s), standbys: x 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.524786+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.525911+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.526710+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.527341+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi100.ppcqpk"}]: dispatch 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.528107+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.nphkis"}]: dispatch 2024-07-28T19:35:38.042 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.528833+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.529518+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.529871+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.530543+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.530844+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.531123+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.531431+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.531755+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.532009+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.532392+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-07-28T19:35:38.043 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.532788+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-07-28T19:35:38.044 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.533097+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-07-28T19:35:38.044 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.533693+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-07-28T19:35:38.044 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: cluster 2024-07-28T19:35:37.541122+0000 mon.a (mon.0) 46 : cluster [INF] Manager daemon y is now available 2024-07-28T19:35:38.044 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.566179+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:38.044 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.611118+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:38.044 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.615558+0000 mon.a (mon.0) 49 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:38.044 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.620011+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-07-28T19:35:38.044 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:37 smithi138 bash[23952]: audit 2024-07-28T19:35:37.645155+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-07-28T19:35:38.829 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:38 smithi100 bash[73305]: debug 2024-07-28T19:35:38.526+0000 7f9676ff2640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: debug 2024-07-28T19:35:39.114+0000 7f962b69c640 -1 log_channel(cephadm) log [ERR] : [28/Jul/2024:19:35:39] ENGINE Error in HTTPServer.serve 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: Traceback (most recent call last): 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: self._connections.run(self.expiration_interval) 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: self._run(expiration_interval) 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:35:39.507 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: s = self.context.wrap_socket( 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: return self.sslsocket_class._create( 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: self.do_handshake() 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: self._sslobj.do_handshake() 2024-07-28T19:35:39.508 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[73305]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:35:39.635 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:39.722 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:39.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: cluster 2024-07-28T19:35:38.525785+0000 mon.a (mon.0) 52 : cluster [DBG] mgrmap e36: y(active, since 1.02494s), standbys: x 2024-07-28T19:35:39.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: cluster 2024-07-28T19:35:38.616221+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-07-28T19:35:39.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: cephadm 2024-07-28T19:35:38.997965+0000 mgr.y (mgr.25358) 2 : cephadm [INF] [28/Jul/2024:19:35:38] ENGINE Bus STARTING 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: cephadm 2024-07-28T19:35:39.114747+0000 mgr.y (mgr.25358) 3 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: cephadm 2024-07-28T19:35:39.118226+0000 mgr.y (mgr.25358) 4 : cephadm [ERR] [28/Jul/2024:19:35:39] ENGINE Error in HTTPServer.serve 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: Traceback (most recent call last): 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: self._connections.run(self.expiration_interval) 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: self._run(expiration_interval) 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:35:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: s = self.context.wrap_socket( 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: return self.sslsocket_class._create( 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: self.do_handshake() 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: self._sslobj.do_handshake() 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: cephadm 2024-07-28T19:35:39.216159+0000 mgr.y (mgr.25358) 5 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Serving on http://172.21.15.100:8765 2024-07-28T19:35:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:39 smithi138 bash[23952]: cephadm 2024-07-28T19:35:39.216307+0000 mgr.y (mgr.25358) 6 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Bus STARTED 2024-07-28T19:35:39.878 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: cluster 2024-07-28T19:35:38.525785+0000 mon.a (mon.0) 52 : cluster [DBG] mgrmap e36: y(active, since 1.02494s), standbys: x 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: cluster 2024-07-28T19:35:38.616221+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: cephadm 2024-07-28T19:35:38.997965+0000 mgr.y (mgr.25358) 2 : cephadm [INF] [28/Jul/2024:19:35:38] ENGINE Bus STARTING 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: cephadm 2024-07-28T19:35:39.114747+0000 mgr.y (mgr.25358) 3 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: cephadm 2024-07-28T19:35:39.118226+0000 mgr.y (mgr.25358) 4 : cephadm [ERR] [28/Jul/2024:19:35:39] ENGINE Error in HTTPServer.serve 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: Traceback (most recent call last): 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: self._connections.run(self.expiration_interval) 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: self._run(expiration_interval) 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:35:39.879 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: s = self.context.wrap_socket( 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: return self.sslsocket_class._create( 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: self.do_handshake() 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: self._sslobj.do_handshake() 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: cephadm 2024-07-28T19:35:39.216159+0000 mgr.y (mgr.25358) 5 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Serving on http://172.21.15.100:8765 2024-07-28T19:35:39.880 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[31715]: cephadm 2024-07-28T19:35:39.216307+0000 mgr.y (mgr.25358) 6 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Bus STARTED 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: cluster 2024-07-28T19:35:38.525785+0000 mon.a (mon.0) 52 : cluster [DBG] mgrmap e36: y(active, since 1.02494s), standbys: x 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: cluster 2024-07-28T19:35:38.616221+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: cephadm 2024-07-28T19:35:38.997965+0000 mgr.y (mgr.25358) 2 : cephadm [INF] [28/Jul/2024:19:35:38] ENGINE Bus STARTING 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: cephadm 2024-07-28T19:35:39.114747+0000 mgr.y (mgr.25358) 3 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Serving on https://172.21.15.100:7150 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: cephadm 2024-07-28T19:35:39.118226+0000 mgr.y (mgr.25358) 4 : cephadm [ERR] [28/Jul/2024:19:35:39] ENGINE Error in HTTPServer.serve 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: Traceback (most recent call last): 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: self._connections.run(self.expiration_interval) 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-07-28T19:35:39.881 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: self._run(expiration_interval) 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: new_conn = self._from_server_socket(self.server.socket) 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: s = self.context.wrap_socket( 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: return self.sslsocket_class._create( 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: self.do_handshake() 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: self._sslobj.do_handshake() 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: cephadm 2024-07-28T19:35:39.216159+0000 mgr.y (mgr.25358) 5 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Serving on http://172.21.15.100:8765 2024-07-28T19:35:39.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:39 smithi100 bash[80199]: cephadm 2024-07-28T19:35:39.216307+0000 mgr.y (mgr.25358) 6 : cephadm [INF] [28/Jul/2024:19:35:39] ENGINE Bus STARTED 2024-07-28T19:35:40.945 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:40 smithi100 bash[31715]: cluster 2024-07-28T19:35:39.531054+0000 mgr.y (mgr.25358) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:40.946 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:40 smithi100 bash[31715]: cluster 2024-07-28T19:35:39.618032+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-07-28T19:35:40.946 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:40 smithi100 bash[31715]: cluster 2024-07-28T19:35:39.765541+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-07-28T19:35:40.946 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:40 smithi100 bash[31715]: cluster 2024-07-28T19:35:40.532323+0000 mon.a (mon.0) 56 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2024-07-28T19:35:40.946 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:40 smithi100 bash[80199]: cluster 2024-07-28T19:35:39.531054+0000 mgr.y (mgr.25358) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:40.946 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:40 smithi100 bash[80199]: cluster 2024-07-28T19:35:39.618032+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-07-28T19:35:40.946 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:40 smithi100 bash[80199]: cluster 2024-07-28T19:35:39.765541+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-07-28T19:35:40.946 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:40 smithi100 bash[80199]: cluster 2024-07-28T19:35:40.532323+0000 mon.a (mon.0) 56 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2024-07-28T19:35:41.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:40 smithi138 bash[23952]: cluster 2024-07-28T19:35:39.531054+0000 mgr.y (mgr.25358) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:41.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:40 smithi138 bash[23952]: cluster 2024-07-28T19:35:39.618032+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-07-28T19:35:41.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:40 smithi138 bash[23952]: cluster 2024-07-28T19:35:39.765541+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-07-28T19:35:41.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:40 smithi138 bash[23952]: cluster 2024-07-28T19:35:40.532323+0000 mon.a (mon.0) 56 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2024-07-28T19:35:41.055 INFO:tasks.workunit.client.0.smithi100.stdout:migration_abort 2024-07-28T19:35:41.335 INFO:teuthology.orchestra.run.smithi100.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:alertmanager.a smithi100 *:9093,9094 running (101s) 59s ago 8m 13.6M - 0.25.0 c8568f914cd2 61edf7ffcf9d 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:grafana.a smithi138 *:3000 running (89s) 30s ago 8m 75.5M - 9.4.7 954c08fa6188 fe0244c1bc8a 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi100.ppcqpk smithi100 running (6m) 59s ago 6m 13.0M - 17.2.7-1155-g3120d060 5ec745195f7e 24cab9a6a84c 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi138.nphkis smithi138 running (6m) 30s ago 6m 15.8M - 17.2.7-1155-g3120d060 5ec745195f7e cc36fa0642f5 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.x smithi138 *:8443,9283,8765 running (36s) 30s ago 17m 106M - 18.2.2-1765-g1c7dd747 dd66d6cc220c 6027bd7b0eab 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.y smithi100 *:8443,9283,8765 running (82s) 59s ago 19m 457M - 18.2.2-1765-g1c7dd747 dd66d6cc220c d4cdb32af700 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:mon.a smithi100 running (19m) 59s ago 19m 56.8M 2048M 17.2.7-1155-g3120d060 5ec745195f7e c0ba6296a8a2 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:mon.b smithi138 running (17m) 30s ago 17m 46.4M 2048M 17.2.7-1155-g3120d060 5ec745195f7e a76f97e1f96b 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:mon.c smithi100 running (17m) 59s ago 17m 48.6M 2048M 17.2.7-1155-g3120d060 5ec745195f7e 59bb2850a096 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.a smithi100 *:9100 running (108s) 59s ago 9m 7711k - 1.5.0 0da6a335fe13 558573030b79 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.b smithi138 *:9100 running (94s) 30s ago 9m 7988k - 1.5.0 0da6a335fe13 3867a7654990 2024-07-28T19:35:41.336 INFO:teuthology.orchestra.run.smithi100.stdout:osd.0 smithi100 running (16m) 59s ago 16m 106M 2524M 17.2.7-1155-g3120d060 5ec745195f7e bc95b51f52b0 2024-07-28T19:35:41.337 INFO:teuthology.orchestra.run.smithi100.stdout:osd.1 smithi100 running (15m) 59s ago 15m 99.7M 2524M 17.2.7-1155-g3120d060 5ec745195f7e ceb5589accd5 2024-07-28T19:35:41.337 INFO:teuthology.orchestra.run.smithi100.stdout:osd.2 smithi100 running (14m) 59s ago 14m 115M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 1b4df5053189 2024-07-28T19:35:41.337 INFO:teuthology.orchestra.run.smithi100.stdout:osd.3 smithi100 running (13m) 59s ago 13m 102M 2524M 17.2.7-1155-g3120d060 5ec745195f7e cf8d59f38b6d 2024-07-28T19:35:41.337 INFO:teuthology.orchestra.run.smithi100.stdout:osd.4 smithi138 running (12m) 30s ago 12m 101M 2524M 17.2.7-1155-g3120d060 5ec745195f7e feff0c636bab 2024-07-28T19:35:41.337 INFO:teuthology.orchestra.run.smithi100.stdout:osd.5 smithi138 running (11m) 30s ago 11m 96.6M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 33bea4a7c9ec 2024-07-28T19:35:41.337 INFO:teuthology.orchestra.run.smithi100.stdout:osd.6 smithi138 running (11m) 30s ago 11m 130M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 11e0eaf60c0b 2024-07-28T19:35:41.337 INFO:teuthology.orchestra.run.smithi100.stdout:osd.7 smithi138 running (10m) 30s ago 10m 139M 2524M 17.2.7-1155-g3120d060 5ec745195f7e fd79c6c44f0f 2024-07-28T19:35:41.337 INFO:teuthology.orchestra.run.smithi100.stdout:prometheus.a smithi138 *:9095 running (43s) 30s ago 9m 41.6M - 2.43.0 a07b618ecd1d bc318a7bc21e 2024-07-28T19:35:42.039 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (4366 ms) 2024-07-28T19:35:42.039 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-07-28T19:35:42.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:41 smithi138 bash[23952]: cluster 2024-07-28T19:35:40.768478+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-07-28T19:35:42.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:41 smithi138 bash[23952]: audit 2024-07-28T19:35:41.304769+0000 mgr.y (mgr.25358) 8 : audit [DBG] from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:35:42.101 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:41 smithi100 bash[31715]: cluster 2024-07-28T19:35:40.768478+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-07-28T19:35:42.102 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:41 smithi100 bash[31715]: audit 2024-07-28T19:35:41.304769+0000 mgr.y (mgr.25358) 8 : audit [DBG] from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:35:42.102 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:41 smithi100 bash[80199]: cluster 2024-07-28T19:35:40.768478+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-07-28T19:35:42.102 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:41 smithi100 bash[80199]: audit 2024-07-28T19:35:41.304769+0000 mgr.y (mgr.25358) 8 : audit [DBG] from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:35:42.103 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch upgrade pause' 2024-07-28T19:35:42.116 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:42.141 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 15859892~1453951 2024-07-28T19:35:42.175 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 52511073~1860431 2024-07-28T19:35:42.176 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 5136544~6908894 2024-07-28T19:35:42.256 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 59826304~4853761 2024-07-28T19:35:42.256 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 78023623~3736770 2024-07-28T19:35:42.291 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 11830740~7027537 2024-07-28T19:35:42.308 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 80436703~3449377 2024-07-28T19:35:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 80083278~1335104 2024-07-28T19:35:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap0 2024-07-28T19:35:42.785 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:42 smithi138 bash[60615]: ts=2024-07-28T19:35:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:43.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:42 smithi138 bash[23952]: cluster 2024-07-28T19:35:41.531742+0000 mgr.y (mgr.25358) 9 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:43.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:42 smithi138 bash[23952]: cluster 2024-07-28T19:35:41.772037+0000 mon.a (mon.0) 58 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-07-28T19:35:43.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:42 smithi138 bash[23952]: cluster 2024-07-28T19:35:41.793962+0000 mon.a (mon.0) 59 : cluster [DBG] mgrmap e38: y(active, since 4s), standbys: x 2024-07-28T19:35:43.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:35:42] "GET /metrics HTTP/1.1" 200 36066 "" "Prometheus/2.43.0" 2024-07-28T19:35:43.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:42 smithi100 bash[31715]: cluster 2024-07-28T19:35:41.531742+0000 mgr.y (mgr.25358) 9 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:43.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:42 smithi100 bash[31715]: cluster 2024-07-28T19:35:41.772037+0000 mon.a (mon.0) 58 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-07-28T19:35:43.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:42 smithi100 bash[31715]: cluster 2024-07-28T19:35:41.793962+0000 mon.a (mon.0) 59 : cluster [DBG] mgrmap e38: y(active, since 4s), standbys: x 2024-07-28T19:35:43.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:42 smithi100 bash[80199]: cluster 2024-07-28T19:35:41.531742+0000 mgr.y (mgr.25358) 9 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:35:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:42 smithi100 bash[80199]: cluster 2024-07-28T19:35:41.772037+0000 mon.a (mon.0) 58 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-07-28T19:35:43.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:42 smithi100 bash[80199]: cluster 2024-07-28T19:35:41.793962+0000 mon.a (mon.0) 59 : cluster [DBG] mgrmap e38: y(active, since 4s), standbys: x 2024-07-28T19:35:43.793 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:43 smithi138 bash[60615]: ts=2024-07-28T19:35:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:43.811 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap0 2024-07-28T19:35:43.834 INFO:tasks.workunit.client.0.smithi100.stdout:clone image157 -> image160 2024-07-28T19:35:43.949 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 56782476~6366277 2024-07-28T19:35:44.006 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 4821305~6227004 2024-07-28T19:35:44.035 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 14144116~5299059 2024-07-28T19:35:44.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:43 smithi138 bash[23952]: cluster 2024-07-28T19:35:42.785060+0000 mon.a (mon.0) 60 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-07-28T19:35:44.076 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 47362970~4067630 2024-07-28T19:35:44.084 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 60666919~8095436 2024-07-28T19:35:44.133 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 50529542~5193602 2024-07-28T19:35:44.140 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 56979202~8241996 2024-07-28T19:35:44.189 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 76216902~7669178 2024-07-28T19:35:44.202 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:44.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:43 smithi100 bash[80199]: cluster 2024-07-28T19:35:42.785060+0000 mon.a (mon.0) 60 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-07-28T19:35:44.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:43 smithi100 bash[31715]: cluster 2024-07-28T19:35:42.785060+0000 mon.a (mon.0) 60 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-07-28T19:35:44.795 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[60615]: ts=2024-07-28T19:35:44.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:44.795 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[60615]: ts=2024-07-28T19:35:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:45.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[23952]: cluster 2024-07-28T19:35:43.532660+0000 mgr.y (mgr.25358) 10 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 904 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s rd, 1.1 MiB/s wr, 52 op/s 2024-07-28T19:35:45.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[23952]: cluster 2024-07-28T19:35:43.808092+0000 mon.a (mon.0) 61 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-07-28T19:35:45.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[23952]: audit 2024-07-28T19:35:43.926060+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[23952]: audit 2024-07-28T19:35:43.931136+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[23952]: audit 2024-07-28T19:35:44.372009+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[23952]: audit 2024-07-28T19:35:44.379809+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[23952]: audit 2024-07-28T19:35:44.632208+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:44 smithi138 bash[23952]: audit 2024-07-28T19:35:44.637850+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[80199]: cluster 2024-07-28T19:35:43.532660+0000 mgr.y (mgr.25358) 10 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 904 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s rd, 1.1 MiB/s wr, 52 op/s 2024-07-28T19:35:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[80199]: cluster 2024-07-28T19:35:43.808092+0000 mon.a (mon.0) 61 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-07-28T19:35:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[80199]: audit 2024-07-28T19:35:43.926060+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[80199]: audit 2024-07-28T19:35:43.931136+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[80199]: audit 2024-07-28T19:35:44.372009+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[80199]: audit 2024-07-28T19:35:44.379809+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[80199]: audit 2024-07-28T19:35:44.632208+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[80199]: audit 2024-07-28T19:35:44.637850+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[31715]: cluster 2024-07-28T19:35:43.532660+0000 mgr.y (mgr.25358) 10 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 904 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s rd, 1.1 MiB/s wr, 52 op/s 2024-07-28T19:35:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[31715]: cluster 2024-07-28T19:35:43.808092+0000 mon.a (mon.0) 61 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-07-28T19:35:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[31715]: audit 2024-07-28T19:35:43.926060+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[31715]: audit 2024-07-28T19:35:43.931136+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[31715]: audit 2024-07-28T19:35:44.372009+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[31715]: audit 2024-07-28T19:35:44.379809+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[31715]: audit 2024-07-28T19:35:44.632208+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:44 smithi100 bash[31715]: audit 2024-07-28T19:35:44.637850+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:45.973 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:46.002 INFO:tasks.workunit.client.0.smithi100.stdout:clone image160 -> image162 2024-07-28T19:35:46.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:45 smithi100 bash[80199]: cluster 2024-07-28T19:35:44.937648+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-07-28T19:35:46.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:45 smithi100 bash[80199]: audit 2024-07-28T19:35:45.045632+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:46.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:45 smithi100 bash[80199]: audit 2024-07-28T19:35:45.050579+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:46.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:45 smithi100 bash[31715]: cluster 2024-07-28T19:35:44.937648+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-07-28T19:35:46.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:45 smithi100 bash[31715]: audit 2024-07-28T19:35:45.045632+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:46.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:45 smithi100 bash[31715]: audit 2024-07-28T19:35:45.050579+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:45 smithi138 bash[23952]: cluster 2024-07-28T19:35:44.937648+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-07-28T19:35:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:45 smithi138 bash[23952]: audit 2024-07-28T19:35:45.045632+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:46.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:45 smithi138 bash[23952]: audit 2024-07-28T19:35:45.050579+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:46.311 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 22069348~6850363 2024-07-28T19:35:46.395 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 52530173~7295512 2024-07-28T19:35:46.462 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 64654698~7818283 2024-07-28T19:35:46.563 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 36433485~5640658 2024-07-28T19:35:46.578 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 80124678~3761402 2024-07-28T19:35:46.629 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 9358852~7464859 2024-07-28T19:35:46.650 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:35:46.675 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 22219744~2377288 2024-07-28T19:35:46.697 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 67324548~7421072 2024-07-28T19:35:46.703 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:35:47.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:46 smithi100 bash[80199]: cluster 2024-07-28T19:35:45.533452+0000 mgr.y (mgr.25358) 11 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 931 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 315 KiB/s rd, 7.4 MiB/s wr, 383 op/s 2024-07-28T19:35:47.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:46 smithi100 bash[80199]: cluster 2024-07-28T19:35:45.948986+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-07-28T19:35:47.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:46 smithi100 bash[31715]: cluster 2024-07-28T19:35:45.533452+0000 mgr.y (mgr.25358) 11 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 931 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 315 KiB/s rd, 7.4 MiB/s wr, 383 op/s 2024-07-28T19:35:47.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:46 smithi100 bash[31715]: cluster 2024-07-28T19:35:45.948986+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-07-28T19:35:47.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:46 smithi138 bash[23952]: cluster 2024-07-28T19:35:45.533452+0000 mgr.y (mgr.25358) 11 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 931 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 315 KiB/s rd, 7.4 MiB/s wr, 383 op/s 2024-07-28T19:35:47.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:46 smithi138 bash[23952]: cluster 2024-07-28T19:35:45.948986+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-07-28T19:35:48.088 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap2 2024-07-28T19:35:48.113 INFO:tasks.workunit.client.0.smithi100.stdout:clone image162 -> image164 2024-07-28T19:35:48.370 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:48 smithi100 bash[80199]: cluster 2024-07-28T19:35:47.061459+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-07-28T19:35:48.371 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:48 smithi100 bash[31715]: cluster 2024-07-28T19:35:47.061459+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-07-28T19:35:48.395 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 115023096 2024-07-28T19:35:48.423 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 93782229~5942855 2024-07-28T19:35:48.497 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 68180222~5678020 2024-07-28T19:35:48.516 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 100712714~6556448 2024-07-28T19:35:48.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:48 smithi138 bash[23952]: cluster 2024-07-28T19:35:47.061459+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-07-28T19:35:48.586 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 65106098~7742906 2024-07-28T19:35:48.638 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 55098237~5156485 2024-07-28T19:35:48.724 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 95376725~5253765 2024-07-28T19:35:48.730 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 105527394~7918480 2024-07-28T19:35:48.806 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 46088854~1864246 2024-07-28T19:35:48.824 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:35:49.054 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:48 smithi138 bash[60615]: ts=2024-07-28T19:35:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:49.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:49 smithi100 bash[80199]: cluster 2024-07-28T19:35:47.534269+0000 mgr.y (mgr.25358) 12 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 931 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 273 KiB/s rd, 6.3 MiB/s wr, 331 op/s 2024-07-28T19:35:49.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:49 smithi100 bash[80199]: cluster 2024-07-28T19:35:48.071672+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-07-28T19:35:49.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:49 smithi100 bash[31715]: cluster 2024-07-28T19:35:47.534269+0000 mgr.y (mgr.25358) 12 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 931 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 273 KiB/s rd, 6.3 MiB/s wr, 331 op/s 2024-07-28T19:35:49.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:49 smithi100 bash[31715]: cluster 2024-07-28T19:35:48.071672+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-07-28T19:35:49.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:49 smithi138 bash[23952]: cluster 2024-07-28T19:35:47.534269+0000 mgr.y (mgr.25358) 12 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 931 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 273 KiB/s rd, 6.3 MiB/s wr, 331 op/s 2024-07-28T19:35:49.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:49 smithi138 bash[23952]: cluster 2024-07-28T19:35:48.071672+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-07-28T19:35:49.800 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap3 2024-07-28T19:35:49.826 INFO:tasks.workunit.client.0.smithi100.stdout:clone image164 -> image166 2024-07-28T19:35:50.461 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.StressNoMigrate (8422 ms) 2024-07-28T19:35:50.462 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.Stress 2024-07-28T19:35:50.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:50 smithi100 bash[80199]: cluster 2024-07-28T19:35:49.069404+0000 mon.a (mon.0) 74 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-07-28T19:35:50.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:50 smithi100 bash[80199]: cluster 2024-07-28T19:35:49.773525+0000 mon.a (mon.0) 75 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-07-28T19:35:50.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:50 smithi100 bash[31715]: cluster 2024-07-28T19:35:49.069404+0000 mon.a (mon.0) 74 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-07-28T19:35:50.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:50 smithi100 bash[31715]: cluster 2024-07-28T19:35:49.773525+0000 mon.a (mon.0) 75 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-07-28T19:35:50.498 INFO:teuthology.orchestra.run.smithi100.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:35:50.523 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:35:50.538 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 21243234~7369056 2024-07-28T19:35:50.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:50 smithi138 bash[23952]: cluster 2024-07-28T19:35:49.069404+0000 mon.a (mon.0) 74 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-07-28T19:35:50.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:50 smithi138 bash[23952]: cluster 2024-07-28T19:35:49.773525+0000 mon.a (mon.0) 75 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-07-28T19:35:50.601 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 5956210~3171482 2024-07-28T19:35:50.601 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 20918943~3542269 2024-07-28T19:35:50.627 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 59316445~8333498 2024-07-28T19:35:50.627 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 160574~3174191 2024-07-28T19:35:50.668 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 15894323~2471415 2024-07-28T19:35:50.672 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 16802878~6131766 2024-07-28T19:35:50.721 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 78918650~479952 2024-07-28T19:35:50.722 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap0 2024-07-28T19:35:51.322 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'sleep 60' 2024-07-28T19:35:51.376 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:51 smithi100 bash[80199]: cluster 2024-07-28T19:35:49.535085+0000 mgr.y (mgr.25358) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 634 op/s 2024-07-28T19:35:51.376 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:51 smithi100 bash[80199]: audit 2024-07-28T19:35:50.505434+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:51.376 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:51 smithi100 bash[31715]: cluster 2024-07-28T19:35:49.535085+0000 mgr.y (mgr.25358) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 634 op/s 2024-07-28T19:35:51.376 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:51 smithi100 bash[31715]: audit 2024-07-28T19:35:50.505434+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:51.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:51 smithi138 bash[23952]: cluster 2024-07-28T19:35:49.535085+0000 mgr.y (mgr.25358) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 634 op/s 2024-07-28T19:35:51.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:51 smithi138 bash[23952]: audit 2024-07-28T19:35:50.505434+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:52.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:52 smithi100 bash[80199]: audit 2024-07-28T19:35:50.500198+0000 mgr.y (mgr.25358) 14 : audit [DBG] from='client.25403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:35:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:52 smithi100 bash[80199]: cephadm 2024-07-28T19:35:50.500675+0000 mgr.y (mgr.25358) 15 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:35:52.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:52 smithi100 bash[80199]: cluster 2024-07-28T19:35:51.512037+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-07-28T19:35:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:52 smithi100 bash[31715]: audit 2024-07-28T19:35:50.500198+0000 mgr.y (mgr.25358) 14 : audit [DBG] from='client.25403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:35:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:52 smithi100 bash[31715]: cephadm 2024-07-28T19:35:50.500675+0000 mgr.y (mgr.25358) 15 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:35:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:52 smithi100 bash[31715]: cluster 2024-07-28T19:35:51.512037+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-07-28T19:35:52.539 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap0 2024-07-28T19:35:52.554 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:52 smithi138 bash[60615]: ts=2024-07-28T19:35:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:52.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:52 smithi138 bash[23952]: audit 2024-07-28T19:35:50.500198+0000 mgr.y (mgr.25358) 14 : audit [DBG] from='client.25403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:35:52.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:52 smithi138 bash[23952]: cephadm 2024-07-28T19:35:50.500675+0000 mgr.y (mgr.25358) 15 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:35:52.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:52 smithi138 bash[23952]: cluster 2024-07-28T19:35:51.512037+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-07-28T19:35:52.565 INFO:tasks.workunit.client.0.smithi100.stdout:clone image167 -> image170 2024-07-28T19:35:52.797 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 24046695~908485 2024-07-28T19:35:52.888 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 73173834~3463186 2024-07-28T19:35:52.899 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 42271384~8337597 2024-07-28T19:35:52.974 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 7867469~633703 2024-07-28T19:35:52.984 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 69606813~2485634 2024-07-28T19:35:53.015 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 46015639~1207259 2024-07-28T19:35:53.020 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 8576315~1908029 2024-07-28T19:35:53.040 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 63799768~150010 2024-07-28T19:35:53.050 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:35:53.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:35:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:35:52] "GET /metrics HTTP/1.1" 200 38883 "" "Prometheus/2.43.0" 2024-07-28T19:35:53.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:53 smithi100 bash[80199]: cluster 2024-07-28T19:35:51.536026+0000 mgr.y (mgr.25358) 16 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 634 op/s 2024-07-28T19:35:53.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:53 smithi100 bash[80199]: cluster 2024-07-28T19:35:52.516110+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-07-28T19:35:53.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:53 smithi100 bash[80199]: audit 2024-07-28T19:35:52.605778+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:53 smithi100 bash[31715]: cluster 2024-07-28T19:35:51.536026+0000 mgr.y (mgr.25358) 16 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 634 op/s 2024-07-28T19:35:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:53 smithi100 bash[31715]: cluster 2024-07-28T19:35:52.516110+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-07-28T19:35:53.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:53 smithi100 bash[31715]: audit 2024-07-28T19:35:52.605778+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:53 smithi138 bash[23952]: cluster 2024-07-28T19:35:51.536026+0000 mgr.y (mgr.25358) 16 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 634 op/s 2024-07-28T19:35:53.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:53 smithi138 bash[23952]: cluster 2024-07-28T19:35:52.516110+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-07-28T19:35:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:53 smithi138 bash[23952]: audit 2024-07-28T19:35:52.605778+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:35:53.804 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:53 smithi138 bash[60615]: ts=2024-07-28T19:35:53.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:54.550 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:35:54.576 INFO:tasks.workunit.client.0.smithi100.stdout:clone image170 -> image172 2024-07-28T19:35:54.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:54 smithi138 bash[23952]: cluster 2024-07-28T19:35:53.528340+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-07-28T19:35:54.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:54 smithi138 bash[23952]: cluster 2024-07-28T19:35:53.537267+0000 mgr.y (mgr.25358) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.3 MiB/s rd, 8.8 MiB/s wr, 91 op/s 2024-07-28T19:35:54.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:54 smithi138 bash[60615]: ts=2024-07-28T19:35:54.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:54.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:54 smithi138 bash[60615]: ts=2024-07-28T19:35:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:54.822 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 84718711 2024-07-28T19:35:54.837 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 15299566~6088807 2024-07-28T19:35:54.910 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 19758371~4006822 2024-07-28T19:35:54.913 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 77958827~6759884 2024-07-28T19:35:54.959 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 71458815~3992539 2024-07-28T19:35:54.973 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 81114829~3603882 2024-07-28T19:35:54.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:54 smithi100 bash[80199]: cluster 2024-07-28T19:35:53.528340+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-07-28T19:35:54.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:54 smithi100 bash[80199]: cluster 2024-07-28T19:35:53.537267+0000 mgr.y (mgr.25358) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.3 MiB/s rd, 8.8 MiB/s wr, 91 op/s 2024-07-28T19:35:54.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:54 smithi100 bash[31715]: cluster 2024-07-28T19:35:53.528340+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-07-28T19:35:54.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:54 smithi100 bash[31715]: cluster 2024-07-28T19:35:53.537267+0000 mgr.y (mgr.25358) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.3 MiB/s rd, 8.8 MiB/s wr, 91 op/s 2024-07-28T19:35:54.997 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 23316641~6312119 2024-07-28T19:35:55.005 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 47260627~8322203 2024-07-28T19:35:55.077 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 25398270~5714861 2024-07-28T19:35:55.080 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:35:55.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cluster 2024-07-28T19:35:54.531639+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-07-28T19:35:55.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:54.644596+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:54.655244+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:54.658922+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:55.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:54.660694+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cephadm 2024-07-28T19:35:54.663460+0000 mgr.y (mgr.25358) 18 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cephadm 2024-07-28T19:35:54.665214+0000 mgr.y (mgr.25358) 19 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cephadm 2024-07-28T19:35:54.859507+0000 mgr.y (mgr.25358) 20 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cephadm 2024-07-28T19:35:54.867915+0000 mgr.y (mgr.25358) 21 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cephadm 2024-07-28T19:35:55.015143+0000 mgr.y (mgr.25358) 22 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cephadm 2024-07-28T19:35:55.035973+0000 mgr.y (mgr.25358) 23 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cephadm 2024-07-28T19:35:55.152719+0000 mgr.y (mgr.25358) 24 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: cephadm 2024-07-28T19:35:55.173019+0000 mgr.y (mgr.25358) 25 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.276332+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.281334+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.309310+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.317107+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.324053+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.398457+0000 mon.a (mon.0) 91 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.400306+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.401073+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:55 smithi138 bash[23952]: audit 2024-07-28T19:35:55.406644+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cluster 2024-07-28T19:35:54.531639+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:54.644596+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:54.655244+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:54.658922+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:54.660694+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cephadm 2024-07-28T19:35:54.663460+0000 mgr.y (mgr.25358) 18 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cephadm 2024-07-28T19:35:54.665214+0000 mgr.y (mgr.25358) 19 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cephadm 2024-07-28T19:35:54.859507+0000 mgr.y (mgr.25358) 20 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cephadm 2024-07-28T19:35:54.867915+0000 mgr.y (mgr.25358) 21 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:35:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cephadm 2024-07-28T19:35:55.015143+0000 mgr.y (mgr.25358) 22 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cephadm 2024-07-28T19:35:55.035973+0000 mgr.y (mgr.25358) 23 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cephadm 2024-07-28T19:35:55.152719+0000 mgr.y (mgr.25358) 24 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: cephadm 2024-07-28T19:35:55.173019+0000 mgr.y (mgr.25358) 25 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.276332+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.281334+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.309310+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.317107+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.324053+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.398457+0000 mon.a (mon.0) 91 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.400306+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.401073+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[80199]: audit 2024-07-28T19:35:55.406644+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cluster 2024-07-28T19:35:54.531639+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:54.644596+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:54.655244+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:54.658922+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:54.660694+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cephadm 2024-07-28T19:35:54.663460+0000 mgr.y (mgr.25358) 18 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.conf 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cephadm 2024-07-28T19:35:54.665214+0000 mgr.y (mgr.25358) 19 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cephadm 2024-07-28T19:35:54.859507+0000 mgr.y (mgr.25358) 20 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cephadm 2024-07-28T19:35:54.867915+0000 mgr.y (mgr.25358) 21 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.conf 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cephadm 2024-07-28T19:35:55.015143+0000 mgr.y (mgr.25358) 22 : cephadm [INF] Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cephadm 2024-07-28T19:35:55.035973+0000 mgr.y (mgr.25358) 23 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cephadm 2024-07-28T19:35:55.152719+0000 mgr.y (mgr.25358) 24 : cephadm [INF] Updating smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: cephadm 2024-07-28T19:35:55.173019+0000 mgr.y (mgr.25358) 25 : cephadm [INF] Updating smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/config/ceph.client.admin.keyring 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.276332+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.281334+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.309310+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.317107+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.324053+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.398457+0000 mon.a (mon.0) 91 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:35:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.400306+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:35:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.401073+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:35:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:55 smithi100 bash[31715]: audit 2024-07-28T19:35:55.406644+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:35:56.160 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:35:56.686 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 114081208 2024-07-28T19:35:56.713 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 87385615~4620841 2024-07-28T19:35:56.734 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83072789~741610 2024-07-28T19:35:56.735 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 98341543~4212830 2024-07-28T19:35:56.745 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 99996832~6015808 2024-07-28T19:35:56.785 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 79307198~7729151 2024-07-28T19:35:56.814 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 11959694~8055120 2024-07-28T19:35:56.814 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 72753149~1471317 2024-07-28T19:35:56.816 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 91645387~4179997 2024-07-28T19:35:56.816 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:35:56.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:56 smithi100 bash[80199]: cluster 2024-07-28T19:35:55.538529+0000 mgr.y (mgr.25358) 26 : cluster [DBG] pgmap v29: 137 pgs: 137 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s rd, 29 MiB/s wr, 501 op/s 2024-07-28T19:35:56.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:56 smithi100 bash[80199]: cluster 2024-07-28T19:35:55.660678+0000 mon.a (mon.0) 95 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-07-28T19:35:56.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:56 smithi100 bash[31715]: cluster 2024-07-28T19:35:55.538529+0000 mgr.y (mgr.25358) 26 : cluster [DBG] pgmap v29: 137 pgs: 137 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s rd, 29 MiB/s wr, 501 op/s 2024-07-28T19:35:56.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:56 smithi100 bash[31715]: cluster 2024-07-28T19:35:55.660678+0000 mon.a (mon.0) 95 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-07-28T19:35:57.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:56 smithi138 bash[23952]: cluster 2024-07-28T19:35:55.538529+0000 mgr.y (mgr.25358) 26 : cluster [DBG] pgmap v29: 137 pgs: 137 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s rd, 29 MiB/s wr, 501 op/s 2024-07-28T19:35:57.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:56 smithi138 bash[23952]: cluster 2024-07-28T19:35:55.660678+0000 mon.a (mon.0) 95 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-07-28T19:35:57.886 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:57 smithi100 bash[31715]: cluster 2024-07-28T19:35:56.662575+0000 mon.a (mon.0) 96 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-07-28T19:35:57.886 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:57 smithi100 bash[80199]: cluster 2024-07-28T19:35:56.662575+0000 mon.a (mon.0) 96 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-07-28T19:35:58.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:57 smithi138 bash[23952]: cluster 2024-07-28T19:35:56.662575+0000 mon.a (mon.0) 96 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-07-28T19:35:58.717 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap3 2024-07-28T19:35:58.740 INFO:tasks.workunit.client.0.smithi100.stdout:clone image172 -> image174 2024-07-28T19:35:58.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:58 smithi100 bash[31715]: cluster 2024-07-28T19:35:57.539280+0000 mgr.y (mgr.25358) 27 : cluster [DBG] pgmap v32: 137 pgs: 137 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s rd, 29 MiB/s wr, 503 op/s 2024-07-28T19:35:58.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:58 smithi100 bash[31715]: cluster 2024-07-28T19:35:57.686110+0000 mon.a (mon.0) 97 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-07-28T19:35:58.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:58 smithi100 bash[80199]: cluster 2024-07-28T19:35:57.539280+0000 mgr.y (mgr.25358) 27 : cluster [DBG] pgmap v32: 137 pgs: 137 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s rd, 29 MiB/s wr, 503 op/s 2024-07-28T19:35:58.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:58 smithi100 bash[80199]: cluster 2024-07-28T19:35:57.686110+0000 mon.a (mon.0) 97 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-07-28T19:35:59.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:58 smithi138 bash[23952]: cluster 2024-07-28T19:35:57.539280+0000 mgr.y (mgr.25358) 27 : cluster [DBG] pgmap v32: 137 pgs: 137 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 24 MiB/s rd, 29 MiB/s wr, 503 op/s 2024-07-28T19:35:59.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:58 smithi138 bash[23952]: cluster 2024-07-28T19:35:57.686110+0000 mon.a (mon.0) 97 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-07-28T19:35:59.058 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:35:58 smithi138 bash[60615]: ts=2024-07-28T19:35:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:35:59.133 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:35:59.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:35:59 smithi100 bash[31715]: cluster 2024-07-28T19:35:58.699111+0000 mon.a (mon.0) 98 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-07-28T19:35:59.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:35:59 smithi100 bash[80199]: cluster 2024-07-28T19:35:58.699111+0000 mon.a (mon.0) 98 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-07-28T19:36:00.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:35:59 smithi138 bash[23952]: cluster 2024-07-28T19:35:58.699111+0000 mon.a (mon.0) 98 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-07-28T19:36:00.073 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:36:00.565 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:36:00.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:00 smithi100 bash[31715]: cluster 2024-07-28T19:35:59.540055+0000 mgr.y (mgr.25358) 28 : cluster [DBG] pgmap v35: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5.0 MiB/s rd, 23 MiB/s wr, 329 op/s 2024-07-28T19:36:00.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:00 smithi100 bash[80199]: cluster 2024-07-28T19:35:59.540055+0000 mgr.y (mgr.25358) 28 : cluster [DBG] pgmap v35: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5.0 MiB/s rd, 23 MiB/s wr, 329 op/s 2024-07-28T19:36:01.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:00 smithi138 bash[23952]: cluster 2024-07-28T19:35:59.540055+0000 mgr.y (mgr.25358) 28 : cluster [DBG] pgmap v35: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 5.0 MiB/s rd, 23 MiB/s wr, 329 op/s 2024-07-28T19:36:01.155 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.Stress (10693 ms) 2024-07-28T19:36:01.155 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.Stress2 2024-07-28T19:36:01.234 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:36:01.264 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 59443451~3324630 2024-07-28T19:36:01.334 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 77847212~2519200 2024-07-28T19:36:01.335 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 13159483~7035315 2024-07-28T19:36:01.407 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 29005522~2283335 2024-07-28T19:36:01.407 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 33447724~2097128 2024-07-28T19:36:01.437 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 16528941~593070 2024-07-28T19:36:01.443 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 47156951~6525561 2024-07-28T19:36:01.504 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 75605516~8280564 2024-07-28T19:36:01.505 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap0 2024-07-28T19:36:02.714 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:02 smithi138 bash[60615]: ts=2024-07-28T19:36:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:02.738 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 33835037~1528202 2024-07-28T19:36:02.757 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 34423178~79019 2024-07-28T19:36:02.773 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 8134139~4051723 2024-07-28T19:36:02.776 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 55854688~1041920 2024-07-28T19:36:02.776 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 72330781~3700512 2024-07-28T19:36:02.786 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 64747025~4855131 2024-07-28T19:36:02.786 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 50122802~4137236 2024-07-28T19:36:02.807 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 48599476~6749964 2024-07-28T19:36:02.844 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:36:02.993 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:36:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:36:02] "GET /metrics HTTP/1.1" 200 38883 "" "Prometheus/2.43.0" 2024-07-28T19:36:02.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:02 smithi100 bash[80199]: cluster 2024-07-28T19:36:01.540796+0000 mgr.y (mgr.25358) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 3.4 MiB/s rd, 16 MiB/s wr, 223 op/s 2024-07-28T19:36:02.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:02 smithi100 bash[80199]: cluster 2024-07-28T19:36:01.714542+0000 mon.a (mon.0) 99 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-07-28T19:36:02.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:02 smithi100 bash[31715]: cluster 2024-07-28T19:36:01.540796+0000 mgr.y (mgr.25358) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 3.4 MiB/s rd, 16 MiB/s wr, 223 op/s 2024-07-28T19:36:02.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:02 smithi100 bash[31715]: cluster 2024-07-28T19:36:01.714542+0000 mon.a (mon.0) 99 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-07-28T19:36:03.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:02 smithi138 bash[23952]: cluster 2024-07-28T19:36:01.540796+0000 mgr.y (mgr.25358) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 3.4 MiB/s rd, 16 MiB/s wr, 223 op/s 2024-07-28T19:36:03.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:02 smithi138 bash[23952]: cluster 2024-07-28T19:36:01.714542+0000 mon.a (mon.0) 99 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-07-28T19:36:03.804 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:03 smithi138 bash[23952]: cluster 2024-07-28T19:36:02.718258+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-07-28T19:36:03.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:03 smithi138 bash[60615]: ts=2024-07-28T19:36:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:03.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:03 smithi100 bash[80199]: cluster 2024-07-28T19:36:02.718258+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-07-28T19:36:03.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:03 smithi100 bash[31715]: cluster 2024-07-28T19:36:02.718258+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-07-28T19:36:04.740 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:04 smithi138 bash[60615]: ts=2024-07-28T19:36:04.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:04.740 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:04 smithi138 bash[60615]: ts=2024-07-28T19:36:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:04.755 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:36:04.791 INFO:tasks.workunit.client.0.smithi100.stdout:clone image175 -> image178 2024-07-28T19:36:04.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:04 smithi100 bash[80199]: cluster 2024-07-28T19:36:03.542083+0000 mgr.y (mgr.25358) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 22 MiB/s wr, 718 op/s 2024-07-28T19:36:04.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:04 smithi100 bash[80199]: cluster 2024-07-28T19:36:03.737621+0000 mon.a (mon.0) 101 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-07-28T19:36:04.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:04 smithi100 bash[31715]: cluster 2024-07-28T19:36:03.542083+0000 mgr.y (mgr.25358) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 22 MiB/s wr, 718 op/s 2024-07-28T19:36:04.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:04 smithi100 bash[31715]: cluster 2024-07-28T19:36:03.737621+0000 mon.a (mon.0) 101 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-07-28T19:36:05.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:04 smithi138 bash[23952]: cluster 2024-07-28T19:36:03.542083+0000 mgr.y (mgr.25358) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 22 MiB/s wr, 718 op/s 2024-07-28T19:36:05.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:04 smithi138 bash[23952]: cluster 2024-07-28T19:36:03.737621+0000 mon.a (mon.0) 101 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-07-28T19:36:05.059 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 145645039 2024-07-28T19:36:05.090 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 44374502~4424230 2024-07-28T19:36:05.175 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 125652937~5135831 2024-07-28T19:36:05.175 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 70541418~1440831 2024-07-28T19:36:05.255 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 56258890~2454475 2024-07-28T19:36:05.268 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 79997912~4779512 2024-07-28T19:36:05.329 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 72954460~919539 2024-07-28T19:36:05.336 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 53842994~2861241 2024-07-28T19:36:05.371 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 95405873~574710 2024-07-28T19:36:05.372 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:36:05.993 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:05 smithi100 bash[80199]: cluster 2024-07-28T19:36:04.738120+0000 mon.a (mon.0) 102 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-07-28T19:36:05.993 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:05 smithi100 bash[31715]: cluster 2024-07-28T19:36:04.738120+0000 mon.a (mon.0) 102 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-07-28T19:36:06.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:05 smithi138 bash[23952]: cluster 2024-07-28T19:36:04.738120+0000 mon.a (mon.0) 102 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-07-28T19:36:06.809 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap2 2024-07-28T19:36:06.835 INFO:tasks.workunit.client.0.smithi100.stdout:clone image178 -> image180 2024-07-28T19:36:07.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:06 smithi138 bash[23952]: cluster 2024-07-28T19:36:05.542831+0000 mgr.y (mgr.25358) 31 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 15 MiB/s wr, 1.01k op/s 2024-07-28T19:36:07.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:06 smithi138 bash[23952]: cluster 2024-07-28T19:36:05.782382+0000 mon.a (mon.0) 103 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-07-28T19:36:07.158 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 191632485 2024-07-28T19:36:07.187 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 31241961~2386300 2024-07-28T19:36:07.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:06 smithi100 bash[80199]: cluster 2024-07-28T19:36:05.542831+0000 mgr.y (mgr.25358) 31 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 15 MiB/s wr, 1.01k op/s 2024-07-28T19:36:07.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:06 smithi100 bash[80199]: cluster 2024-07-28T19:36:05.782382+0000 mon.a (mon.0) 103 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-07-28T19:36:07.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:06 smithi100 bash[31715]: cluster 2024-07-28T19:36:05.542831+0000 mgr.y (mgr.25358) 31 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 15 MiB/s wr, 1.01k op/s 2024-07-28T19:36:07.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:06 smithi100 bash[31715]: cluster 2024-07-28T19:36:05.782382+0000 mon.a (mon.0) 103 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-07-28T19:36:07.273 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 51239855~1605217 2024-07-28T19:36:07.298 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 145182177~1658914 2024-07-28T19:36:07.326 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 33019088~7032585 2024-07-28T19:36:07.341 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 92503946~4010281 2024-07-28T19:36:07.402 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 25131700~5416942 2024-07-28T19:36:07.417 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 15403196~5209102 2024-07-28T19:36:07.498 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 36747832~3796208 2024-07-28T19:36:07.504 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:36:08.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:07 smithi138 bash[23952]: cluster 2024-07-28T19:36:06.792762+0000 mon.a (mon.0) 104 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-07-28T19:36:08.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:07 smithi138 bash[23952]: audit 2024-07-28T19:36:07.605725+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:08.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:07 smithi100 bash[80199]: cluster 2024-07-28T19:36:06.792762+0000 mon.a (mon.0) 104 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-07-28T19:36:08.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:07 smithi100 bash[80199]: audit 2024-07-28T19:36:07.605725+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:08.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:07 smithi100 bash[31715]: cluster 2024-07-28T19:36:06.792762+0000 mon.a (mon.0) 104 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-07-28T19:36:08.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:07 smithi100 bash[31715]: audit 2024-07-28T19:36:07.605725+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:08.832 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap3 2024-07-28T19:36:08.861 INFO:tasks.workunit.client.0.smithi100.stdout:clone image180 -> image182 2024-07-28T19:36:09.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:08 smithi138 bash[23952]: cluster 2024-07-28T19:36:07.543577+0000 mgr.y (mgr.25358) 32 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 6.1 MiB/s wr, 288 op/s 2024-07-28T19:36:09.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:08 smithi138 bash[23952]: cluster 2024-07-28T19:36:07.794940+0000 mon.a (mon.0) 106 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-07-28T19:36:09.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:08 smithi138 bash[60615]: ts=2024-07-28T19:36:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:09.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:08 smithi100 bash[80199]: cluster 2024-07-28T19:36:07.543577+0000 mgr.y (mgr.25358) 32 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 6.1 MiB/s wr, 288 op/s 2024-07-28T19:36:09.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:08 smithi100 bash[80199]: cluster 2024-07-28T19:36:07.794940+0000 mon.a (mon.0) 106 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-07-28T19:36:09.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:08 smithi100 bash[31715]: cluster 2024-07-28T19:36:07.543577+0000 mgr.y (mgr.25358) 32 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 6.1 MiB/s wr, 288 op/s 2024-07-28T19:36:09.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:08 smithi100 bash[31715]: cluster 2024-07-28T19:36:07.794940+0000 mon.a (mon.0) 106 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-07-28T19:36:09.283 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 172114537 2024-07-28T19:36:09.332 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:36:10.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:09 smithi100 bash[31715]: cluster 2024-07-28T19:36:08.807974+0000 mon.a (mon.0) 107 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-07-28T19:36:10.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:09 smithi100 bash[80199]: cluster 2024-07-28T19:36:08.807974+0000 mon.a (mon.0) 107 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-07-28T19:36:10.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:09 smithi138 bash[23952]: cluster 2024-07-28T19:36:08.807974+0000 mon.a (mon.0) 107 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-07-28T19:36:10.449 INFO:tasks.workunit.client.0.smithi100.stdout:write: a 71295095~5971670 2024-07-28T19:36:10.555 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 161636238~4925530 2024-07-28T19:36:10.595 INFO:tasks.workunit.client.0.smithi100.stdout:write: a 111311714~5570360 2024-07-28T19:36:10.676 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 80754104~8091354 2024-07-28T19:36:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:write: a 3177737~3128150 2024-07-28T19:36:10.786 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 153071895~6465311 2024-07-28T19:36:10.822 INFO:tasks.workunit.client.0.smithi100.stdout:write: a 139769123~7340913 2024-07-28T19:36:10.905 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 97579987~1116667 2024-07-28T19:36:10.932 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: user0 2024-07-28T19:36:11.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:10 smithi100 bash[80199]: cluster 2024-07-28T19:36:09.544331+0000 mgr.y (mgr.25358) 33 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6.8 MiB/s rd, 18 MiB/s wr, 427 op/s 2024-07-28T19:36:11.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:10 smithi100 bash[31715]: cluster 2024-07-28T19:36:09.544331+0000 mgr.y (mgr.25358) 33 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6.8 MiB/s rd, 18 MiB/s wr, 427 op/s 2024-07-28T19:36:11.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:10 smithi138 bash[23952]: cluster 2024-07-28T19:36:09.544331+0000 mgr.y (mgr.25358) 33 : cluster [DBG] pgmap v48: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6.8 MiB/s rd, 18 MiB/s wr, 427 op/s 2024-07-28T19:36:12.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:12 smithi138 bash[60615]: ts=2024-07-28T19:36:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:12.867 INFO:tasks.workunit.client.0.smithi100.stdout:write: b 163036682~6859120 2024-07-28T19:36:12.910 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 143902545~3776641 2024-07-28T19:36:12.910 INFO:tasks.workunit.client.0.smithi100.stdout:write: b 137588204~5622586 2024-07-28T19:36:12.931 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 26720466~1141170 2024-07-28T19:36:12.934 INFO:tasks.workunit.client.0.smithi100.stdout:write: b 4048941~7069678 2024-07-28T19:36:12.955 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 43691122~3581097 2024-07-28T19:36:13.045 INFO:tasks.workunit.client.0.smithi100.stdout:write: b 18127698~5232536 2024-07-28T19:36:13.050 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 64685084~3969667 2024-07-28T19:36:13.057 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: user1 2024-07-28T19:36:13.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:36:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:36:12] "GET /metrics HTTP/1.1" 200 38875 "" "Prometheus/2.43.0" 2024-07-28T19:36:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:12 smithi100 bash[80199]: cluster 2024-07-28T19:36:11.545062+0000 mgr.y (mgr.25358) 34 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4.7 MiB/s rd, 12 MiB/s wr, 296 op/s 2024-07-28T19:36:13.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:12 smithi100 bash[80199]: cluster 2024-07-28T19:36:11.844031+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-07-28T19:36:13.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:12 smithi100 bash[31715]: cluster 2024-07-28T19:36:11.545062+0000 mgr.y (mgr.25358) 34 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4.7 MiB/s rd, 12 MiB/s wr, 296 op/s 2024-07-28T19:36:13.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:12 smithi100 bash[31715]: cluster 2024-07-28T19:36:11.844031+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-07-28T19:36:13.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:12 smithi138 bash[23952]: cluster 2024-07-28T19:36:11.545062+0000 mgr.y (mgr.25358) 34 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 4.7 MiB/s rd, 12 MiB/s wr, 296 op/s 2024-07-28T19:36:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:12 smithi138 bash[23952]: cluster 2024-07-28T19:36:11.844031+0000 mon.a (mon.0) 108 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-07-28T19:36:13.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:13 smithi138 bash[60615]: ts=2024-07-28T19:36:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:14.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:13 smithi100 bash[80199]: cluster 2024-07-28T19:36:12.853919+0000 mon.a (mon.0) 109 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-07-28T19:36:14.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:13 smithi100 bash[31715]: cluster 2024-07-28T19:36:12.853919+0000 mon.a (mon.0) 109 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-07-28T19:36:14.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:13 smithi138 bash[23952]: cluster 2024-07-28T19:36:12.853919+0000 mon.a (mon.0) 109 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-07-28T19:36:14.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:14 smithi138 bash[60615]: ts=2024-07-28T19:36:14.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:14.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:14 smithi138 bash[60615]: ts=2024-07-28T19:36:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:14.897 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 190475584 2024-07-28T19:36:14.932 INFO:tasks.workunit.client.0.smithi100.stdout:write: c 95383363~7488958 2024-07-28T19:36:14.987 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 144387948~2175015 2024-07-28T19:36:14.987 INFO:tasks.workunit.client.0.smithi100.stdout:write: c 90473884~6236825 2024-07-28T19:36:15.073 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 117722163~3322124 2024-07-28T19:36:15.073 INFO:tasks.workunit.client.0.smithi100.stdout:write: c 9342972~6081934 2024-07-28T19:36:15.077 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 143029591~6864942 2024-07-28T19:36:15.077 INFO:tasks.workunit.client.0.smithi100.stdout:write: c 104997342~787343 2024-07-28T19:36:15.078 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 143706512~336169 2024-07-28T19:36:15.115 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: user2 2024-07-28T19:36:15.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:14 smithi100 bash[80199]: cluster 2024-07-28T19:36:13.546464+0000 mgr.y (mgr.25358) 35 : cluster [DBG] pgmap v52: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 20 MiB/s rd, 41 MiB/s wr, 764 op/s 2024-07-28T19:36:15.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:14 smithi100 bash[80199]: cluster 2024-07-28T19:36:13.866940+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-07-28T19:36:15.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:14 smithi100 bash[31715]: cluster 2024-07-28T19:36:13.546464+0000 mgr.y (mgr.25358) 35 : cluster [DBG] pgmap v52: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 20 MiB/s rd, 41 MiB/s wr, 764 op/s 2024-07-28T19:36:15.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:14 smithi100 bash[31715]: cluster 2024-07-28T19:36:13.866940+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-07-28T19:36:15.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:14 smithi138 bash[23952]: cluster 2024-07-28T19:36:13.546464+0000 mgr.y (mgr.25358) 35 : cluster [DBG] pgmap v52: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 20 MiB/s rd, 41 MiB/s wr, 764 op/s 2024-07-28T19:36:15.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:14 smithi138 bash[23952]: cluster 2024-07-28T19:36:13.866940+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-07-28T19:36:16.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:15 smithi100 bash[80199]: cluster 2024-07-28T19:36:14.870542+0000 mon.a (mon.0) 111 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-07-28T19:36:16.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:15 smithi100 bash[31715]: cluster 2024-07-28T19:36:14.870542+0000 mon.a (mon.0) 111 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-07-28T19:36:16.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:15 smithi138 bash[23952]: cluster 2024-07-28T19:36:14.870542+0000 mon.a (mon.0) 111 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-07-28T19:36:16.913 INFO:tasks.workunit.client.0.smithi100.stdout:write: d 27070975~5958756 2024-07-28T19:36:16.934 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 86821292~568810 2024-07-28T19:36:16.934 INFO:tasks.workunit.client.0.smithi100.stdout:write: d 105168077~4050029 2024-07-28T19:36:16.938 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 60768283~3594325 2024-07-28T19:36:16.938 INFO:tasks.workunit.client.0.smithi100.stdout:write: d 141981418~2350117 2024-07-28T19:36:16.940 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 46211755~6017694 2024-07-28T19:36:16.941 INFO:tasks.workunit.client.0.smithi100.stdout:write: d 151555883~6089818 2024-07-28T19:36:17.047 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 5734685~3369065 2024-07-28T19:36:17.055 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: user3 2024-07-28T19:36:17.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:16 smithi100 bash[80199]: cluster 2024-07-28T19:36:15.547217+0000 mgr.y (mgr.25358) 36 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 51 MiB/s wr, 775 op/s 2024-07-28T19:36:17.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:16 smithi100 bash[80199]: cluster 2024-07-28T19:36:15.892172+0000 mon.a (mon.0) 112 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-07-28T19:36:17.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:16 smithi100 bash[31715]: cluster 2024-07-28T19:36:15.547217+0000 mgr.y (mgr.25358) 36 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 51 MiB/s wr, 775 op/s 2024-07-28T19:36:17.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:16 smithi100 bash[31715]: cluster 2024-07-28T19:36:15.892172+0000 mon.a (mon.0) 112 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-07-28T19:36:17.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:16 smithi138 bash[23952]: cluster 2024-07-28T19:36:15.547217+0000 mgr.y (mgr.25358) 36 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 51 MiB/s wr, 775 op/s 2024-07-28T19:36:17.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:16 smithi138 bash[23952]: cluster 2024-07-28T19:36:15.892172+0000 mon.a (mon.0) 112 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-07-28T19:36:18.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:17 smithi100 bash[80199]: cluster 2024-07-28T19:36:16.893562+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-07-28T19:36:18.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:17 smithi100 bash[31715]: cluster 2024-07-28T19:36:16.893562+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-07-28T19:36:18.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:17 smithi138 bash[23952]: cluster 2024-07-28T19:36:16.893562+0000 mon.a (mon.0) 113 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-07-28T19:36:18.960 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 19047558~4761889 2024-07-28T19:36:18.969 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 20238030~2380944 2024-07-28T19:36:19.042 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 38095116~4761889 2024-07-28T19:36:19.045 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 39285588~2380944 2024-07-28T19:36:19.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:18 smithi138 bash[23952]: cluster 2024-07-28T19:36:17.547987+0000 mgr.y (mgr.25358) 37 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 6.3 MiB/s rd, 10 MiB/s wr, 103 op/s 2024-07-28T19:36:19.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:18 smithi138 bash[23952]: cluster 2024-07-28T19:36:17.923407+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-07-28T19:36:19.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:18 smithi138 bash[60615]: ts=2024-07-28T19:36:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:19.119 INFO:tasks.workunit.client.0.smithi100.stdout:write: 3 57142675~4761889 2024-07-28T19:36:19.122 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 58333147~2380944 2024-07-28T19:36:19.211 INFO:tasks.workunit.client.0.smithi100.stdout:write: 4 76190233~4761889 2024-07-28T19:36:19.214 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 77380705~2380944 2024-07-28T19:36:19.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:18 smithi100 bash[31715]: cluster 2024-07-28T19:36:17.547987+0000 mgr.y (mgr.25358) 37 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 6.3 MiB/s rd, 10 MiB/s wr, 103 op/s 2024-07-28T19:36:19.243 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:18 smithi100 bash[31715]: cluster 2024-07-28T19:36:17.923407+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-07-28T19:36:19.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:18 smithi100 bash[80199]: cluster 2024-07-28T19:36:17.547987+0000 mgr.y (mgr.25358) 37 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 6.3 MiB/s rd, 10 MiB/s wr, 103 op/s 2024-07-28T19:36:19.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:18 smithi100 bash[80199]: cluster 2024-07-28T19:36:17.923407+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-07-28T19:36:19.280 INFO:tasks.workunit.client.0.smithi100.stdout:write: 5 95237792~4761889 2024-07-28T19:36:19.283 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 96428264~2380944 2024-07-28T19:36:20.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:19 smithi100 bash[80199]: cluster 2024-07-28T19:36:18.928935+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-07-28T19:36:20.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:19 smithi100 bash[31715]: cluster 2024-07-28T19:36:18.928935+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-07-28T19:36:20.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:19 smithi138 bash[23952]: cluster 2024-07-28T19:36:18.928935+0000 mon.a (mon.0) 115 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-07-28T19:36:21.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:20 smithi100 bash[80199]: cluster 2024-07-28T19:36:19.550123+0000 mgr.y (mgr.25358) 38 : cluster [DBG] pgmap v61: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 356 op/s 2024-07-28T19:36:21.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:20 smithi100 bash[31715]: cluster 2024-07-28T19:36:19.550123+0000 mgr.y (mgr.25358) 38 : cluster [DBG] pgmap v61: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 356 op/s 2024-07-28T19:36:21.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:20 smithi138 bash[23952]: cluster 2024-07-28T19:36:19.550123+0000 mgr.y (mgr.25358) 38 : cluster [DBG] pgmap v61: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 356 op/s 2024-07-28T19:36:21.952 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:36:22.549 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:36:22.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:22 smithi138 bash[60615]: ts=2024-07-28T19:36:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:23.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:36:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:36:22] "GET /metrics HTTP/1.1" 200 38880 "" "Prometheus/2.43.0" 2024-07-28T19:36:23.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:22 smithi100 bash[80199]: cluster 2024-07-28T19:36:21.550905+0000 mgr.y (mgr.25358) 39 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.3 MiB/s rd, 21 MiB/s wr, 251 op/s 2024-07-28T19:36:23.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:22 smithi100 bash[80199]: audit 2024-07-28T19:36:22.606705+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:23.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:22 smithi100 bash[31715]: cluster 2024-07-28T19:36:21.550905+0000 mgr.y (mgr.25358) 39 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.3 MiB/s rd, 21 MiB/s wr, 251 op/s 2024-07-28T19:36:23.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:22 smithi100 bash[31715]: audit 2024-07-28T19:36:22.606705+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:23.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:22 smithi138 bash[23952]: cluster 2024-07-28T19:36:21.550905+0000 mgr.y (mgr.25358) 39 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.3 MiB/s rd, 21 MiB/s wr, 251 op/s 2024-07-28T19:36:23.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:22 smithi138 bash[23952]: audit 2024-07-28T19:36:22.606705+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:23.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:23 smithi138 bash[60615]: ts=2024-07-28T19:36:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:24.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:24 smithi138 bash[60615]: ts=2024-07-28T19:36:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:24.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:24 smithi138 bash[60615]: ts=2024-07-28T19:36:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:25.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:24 smithi100 bash[80199]: cluster 2024-07-28T19:36:23.552662+0000 mgr.y (mgr.25358) 40 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 155 MiB/s rd, 25 MiB/s wr, 629 op/s 2024-07-28T19:36:25.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:24 smithi100 bash[31715]: cluster 2024-07-28T19:36:23.552662+0000 mgr.y (mgr.25358) 40 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 155 MiB/s rd, 25 MiB/s wr, 629 op/s 2024-07-28T19:36:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:24 smithi138 bash[23952]: cluster 2024-07-28T19:36:23.552662+0000 mgr.y (mgr.25358) 40 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 155 MiB/s rd, 25 MiB/s wr, 629 op/s 2024-07-28T19:36:25.430 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.Stress2 (24275 ms) 2024-07-28T19:36:25.430 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMigration.StressLive 2024-07-28T19:36:25.514 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 83886080 2024-07-28T19:36:25.544 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 32212534~225877 2024-07-28T19:36:25.570 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 79045473~1017391 2024-07-28T19:36:25.570 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 65819581~4406378 2024-07-28T19:36:25.624 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 11271320~578974 2024-07-28T19:36:25.624 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 34920749~2597780 2024-07-28T19:36:25.661 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 2933949~8218167 2024-07-28T19:36:25.661 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 66366063~5930301 2024-07-28T19:36:25.718 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 78408124~504097 2024-07-28T19:36:25.718 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap0 2024-07-28T19:36:26.997 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap0 2024-07-28T19:36:27.028 INFO:tasks.workunit.client.0.smithi100.stdout:clone image183 -> image186 2024-07-28T19:36:27.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:26 smithi100 bash[80199]: cluster 2024-07-28T19:36:25.553517+0000 mgr.y (mgr.25358) 41 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s rd, 22 MiB/s wr, 582 op/s 2024-07-28T19:36:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:26 smithi100 bash[80199]: cluster 2024-07-28T19:36:25.959333+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-07-28T19:36:27.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:26 smithi100 bash[31715]: cluster 2024-07-28T19:36:25.553517+0000 mgr.y (mgr.25358) 41 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s rd, 22 MiB/s wr, 582 op/s 2024-07-28T19:36:27.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:26 smithi100 bash[31715]: cluster 2024-07-28T19:36:25.959333+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-07-28T19:36:27.283 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 47275239~6960681 2024-07-28T19:36:27.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:26 smithi138 bash[23952]: cluster 2024-07-28T19:36:25.553517+0000 mgr.y (mgr.25358) 41 : cluster [DBG] pgmap v64: 137 pgs: 137 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 153 MiB/s rd, 22 MiB/s wr, 582 op/s 2024-07-28T19:36:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:26 smithi138 bash[23952]: cluster 2024-07-28T19:36:25.959333+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-07-28T19:36:27.372 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 59340356~4111651 2024-07-28T19:36:27.404 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 6490958~7682316 2024-07-28T19:36:27.472 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 70671287~6637795 2024-07-28T19:36:27.533 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 79693912~4192168 2024-07-28T19:36:27.582 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 13390661~6931488 2024-07-28T19:36:27.594 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 15545974~626913 2024-07-28T19:36:27.605 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 18421520~3660155 2024-07-28T19:36:27.619 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap1 2024-07-28T19:36:28.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:27 smithi100 bash[80199]: cluster 2024-07-28T19:36:26.974377+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-07-28T19:36:28.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:27 smithi100 bash[31715]: cluster 2024-07-28T19:36:26.974377+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-07-28T19:36:28.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:27 smithi138 bash[23952]: cluster 2024-07-28T19:36:26.974377+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-07-28T19:36:28.975 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:28 smithi138 bash[60615]: ts=2024-07-28T19:36:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:29.023 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap1 2024-07-28T19:36:29.054 INFO:tasks.workunit.client.0.smithi100.stdout:clone image186 -> image188 2024-07-28T19:36:29.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:28 smithi100 bash[80199]: cluster 2024-07-28T19:36:27.554514+0000 mgr.y (mgr.25358) 42 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s rd, 7.2 MiB/s wr, 404 op/s 2024-07-28T19:36:29.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:28 smithi100 bash[80199]: cluster 2024-07-28T19:36:27.989940+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-07-28T19:36:29.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:28 smithi100 bash[31715]: cluster 2024-07-28T19:36:27.554514+0000 mgr.y (mgr.25358) 42 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s rd, 7.2 MiB/s wr, 404 op/s 2024-07-28T19:36:29.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:28 smithi100 bash[31715]: cluster 2024-07-28T19:36:27.989940+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-07-28T19:36:29.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:28 smithi138 bash[23952]: cluster 2024-07-28T19:36:27.554514+0000 mgr.y (mgr.25358) 42 : cluster [DBG] pgmap v67: 137 pgs: 137 active+clean; 1.8 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s rd, 7.2 MiB/s wr, 404 op/s 2024-07-28T19:36:29.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:28 smithi138 bash[23952]: cluster 2024-07-28T19:36:27.989940+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-07-28T19:36:29.378 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 84241101 2024-07-28T19:36:29.407 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 65409250~2895733 2024-07-28T19:36:29.487 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 6970322~2952801 2024-07-28T19:36:29.548 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 33050804~2390197 2024-07-28T19:36:29.632 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 15584444~1803743 2024-07-28T19:36:29.659 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 44172288~6390848 2024-07-28T19:36:29.750 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 6908174~1473405 2024-07-28T19:36:29.756 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 56072189~3286469 2024-07-28T19:36:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 14293407~2631217 2024-07-28T19:36:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap2 2024-07-28T19:36:30.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:29 smithi138 bash[23952]: cluster 2024-07-28T19:36:28.999497+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-07-28T19:36:30.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:30 smithi100 bash[80199]: cluster 2024-07-28T19:36:28.999497+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-07-28T19:36:30.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:30 smithi100 bash[31715]: cluster 2024-07-28T19:36:28.999497+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-07-28T19:36:31.032 INFO:tasks.workunit.client.0.smithi100.stdout:snap_protect: snap2 2024-07-28T19:36:31.060 INFO:tasks.workunit.client.0.smithi100.stdout:clone image188 -> image190 2024-07-28T19:36:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:31 smithi138 bash[23952]: cluster 2024-07-28T19:36:29.555493+0000 mgr.y (mgr.25358) 43 : cluster [DBG] pgmap v70: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s rd, 18 MiB/s wr, 323 op/s 2024-07-28T19:36:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:31 smithi138 bash[23952]: cluster 2024-07-28T19:36:30.007943+0000 mon.a (mon.0) 121 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-07-28T19:36:31.451 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 126919813 2024-07-28T19:36:31.479 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 91746958~1273128 2024-07-28T19:36:31.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:31 smithi100 bash[80199]: cluster 2024-07-28T19:36:29.555493+0000 mgr.y (mgr.25358) 43 : cluster [DBG] pgmap v70: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s rd, 18 MiB/s wr, 323 op/s 2024-07-28T19:36:31.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:31 smithi100 bash[80199]: cluster 2024-07-28T19:36:30.007943+0000 mon.a (mon.0) 121 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-07-28T19:36:31.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:31 smithi100 bash[31715]: cluster 2024-07-28T19:36:29.555493+0000 mgr.y (mgr.25358) 43 : cluster [DBG] pgmap v70: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s rd, 18 MiB/s wr, 323 op/s 2024-07-28T19:36:31.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:31 smithi100 bash[31715]: cluster 2024-07-28T19:36:30.007943+0000 mon.a (mon.0) 121 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-07-28T19:36:31.515 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 43603499~506299 2024-07-28T19:36:31.561 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 59853468~1539180 2024-07-28T19:36:31.605 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 43533361~6973375 2024-07-28T19:36:31.682 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 10797648~2777516 2024-07-28T19:36:31.737 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 112184441~4877261 2024-07-28T19:36:31.737 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 123122553~3797260 2024-07-28T19:36:31.780 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 90854697~6085338 2024-07-28T19:36:31.798 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: snap3 2024-07-28T19:36:32.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:32 smithi138 bash[23952]: cluster 2024-07-28T19:36:31.015797+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-07-28T19:36:32.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:32 smithi100 bash[80199]: cluster 2024-07-28T19:36:31.015797+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-07-28T19:36:32.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:32 smithi100 bash[31715]: cluster 2024-07-28T19:36:31.015797+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-07-28T19:36:32.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:32 smithi138 bash[60615]: ts=2024-07-28T19:36:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:33.070 INFO:tasks.workunit.client.0.smithi100.stdout:migration_prepare 2024-07-28T19:36:33.243 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:36:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:36:32] "GET /metrics HTTP/1.1" 200 38880 "" "Prometheus/2.43.0" 2024-07-28T19:36:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:33 smithi100 bash[31715]: cluster 2024-07-28T19:36:31.556228+0000 mgr.y (mgr.25358) 44 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s rd, 18 MiB/s wr, 323 op/s 2024-07-28T19:36:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:33 smithi100 bash[31715]: cluster 2024-07-28T19:36:32.025890+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-07-28T19:36:33.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:33 smithi100 bash[80199]: cluster 2024-07-28T19:36:31.556228+0000 mgr.y (mgr.25358) 44 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s rd, 18 MiB/s wr, 323 op/s 2024-07-28T19:36:33.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:33 smithi100 bash[80199]: cluster 2024-07-28T19:36:32.025890+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-07-28T19:36:33.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:33 smithi138 bash[23952]: cluster 2024-07-28T19:36:31.556228+0000 mgr.y (mgr.25358) 44 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 175 MiB/s rd, 18 MiB/s wr, 323 op/s 2024-07-28T19:36:33.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:33 smithi138 bash[23952]: cluster 2024-07-28T19:36:32.025890+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-07-28T19:36:33.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:33 smithi138 bash[60615]: ts=2024-07-28T19:36:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:34.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:34 smithi138 bash[23952]: cluster 2024-07-28T19:36:33.037792+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-07-28T19:36:34.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:34 smithi100 bash[31715]: cluster 2024-07-28T19:36:33.037792+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-07-28T19:36:34.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:34 smithi100 bash[80199]: cluster 2024-07-28T19:36:33.037792+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-07-28T19:36:34.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:34 smithi138 bash[60615]: ts=2024-07-28T19:36:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:34.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:34 smithi138 bash[60615]: ts=2024-07-28T19:36:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:35.182 INFO:tasks.workunit.client.0.smithi100.stdout:migration_execute 2024-07-28T19:36:35.182 INFO:tasks.workunit.client.0.smithi100.stdout:write: a 89571171~1474446 2024-07-28T19:36:35.239 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 60546325~1585720 2024-07-28T19:36:35.269 INFO:tasks.workunit.client.0.smithi100.stdout:write: a 8156958~4560290 2024-07-28T19:36:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:35 smithi138 bash[23952]: cluster 2024-07-28T19:36:33.557883+0000 mgr.y (mgr.25358) 45 : cluster [DBG] pgmap v76: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 23 MiB/s wr, 639 op/s 2024-07-28T19:36:35.343 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:35 smithi138 bash[23952]: cluster 2024-07-28T19:36:34.048256+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-07-28T19:36:35.350 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 120080862~1086702 2024-07-28T19:36:35.355 INFO:tasks.workunit.client.0.smithi100.stdout:write: a 114798933~2736504 2024-07-28T19:36:35.378 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 106731574~687991 2024-07-28T19:36:35.383 INFO:tasks.workunit.client.0.smithi100.stdout:write: a 89873036~7149028 2024-07-28T19:36:35.424 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 67883097~7199 2024-07-28T19:36:35.482 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: user0 2024-07-28T19:36:35.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:35 smithi100 bash[31715]: cluster 2024-07-28T19:36:33.557883+0000 mgr.y (mgr.25358) 45 : cluster [DBG] pgmap v76: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 23 MiB/s wr, 639 op/s 2024-07-28T19:36:35.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:35 smithi100 bash[31715]: cluster 2024-07-28T19:36:34.048256+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-07-28T19:36:35.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:35 smithi100 bash[80199]: cluster 2024-07-28T19:36:33.557883+0000 mgr.y (mgr.25358) 45 : cluster [DBG] pgmap v76: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 14 MiB/s rd, 23 MiB/s wr, 639 op/s 2024-07-28T19:36:35.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:35 smithi100 bash[80199]: cluster 2024-07-28T19:36:34.048256+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-07-28T19:36:37.095 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 226948102 2024-07-28T19:36:37.124 INFO:tasks.workunit.client.0.smithi100.stdout:write: b 125279644~399931 2024-07-28T19:36:37.140 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 136483803~6745715 2024-07-28T19:36:37.140 INFO:tasks.workunit.client.0.smithi100.stdout:write: b 167361041~1263422 2024-07-28T19:36:37.141 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 95056940~6309984 2024-07-28T19:36:37.141 INFO:tasks.workunit.client.0.smithi100.stdout:write: b 135946012~2283660 2024-07-28T19:36:37.143 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 3723149~4731385 2024-07-28T19:36:37.146 INFO:tasks.workunit.client.0.smithi100.stdout:write: b 20248931~7590195 2024-07-28T19:36:37.173 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 109839211~6810033 2024-07-28T19:36:37.174 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: user1 2024-07-28T19:36:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:37 smithi138 bash[23952]: cluster 2024-07-28T19:36:35.558518+0000 mgr.y (mgr.25358) 46 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s rd, 20 MiB/s wr, 562 op/s 2024-07-28T19:36:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:37 smithi138 bash[23952]: cluster 2024-07-28T19:36:36.058127+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-07-28T19:36:37.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:37 smithi100 bash[80199]: cluster 2024-07-28T19:36:35.558518+0000 mgr.y (mgr.25358) 46 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s rd, 20 MiB/s wr, 562 op/s 2024-07-28T19:36:37.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:37 smithi100 bash[80199]: cluster 2024-07-28T19:36:36.058127+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-07-28T19:36:37.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:37 smithi100 bash[31715]: cluster 2024-07-28T19:36:35.558518+0000 mgr.y (mgr.25358) 46 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s rd, 20 MiB/s wr, 562 op/s 2024-07-28T19:36:37.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:37 smithi100 bash[31715]: cluster 2024-07-28T19:36:36.058127+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-07-28T19:36:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:38 smithi100 bash[80199]: cluster 2024-07-28T19:36:37.068427+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-07-28T19:36:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:38 smithi100 bash[80199]: audit 2024-07-28T19:36:37.626177+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:38 smithi100 bash[31715]: cluster 2024-07-28T19:36:37.068427+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-07-28T19:36:38.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:38 smithi100 bash[31715]: audit 2024-07-28T19:36:37.626177+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:38.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:38 smithi138 bash[23952]: cluster 2024-07-28T19:36:37.068427+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-07-28T19:36:38.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:38 smithi138 bash[23952]: audit 2024-07-28T19:36:37.626177+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:39.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:38 smithi138 bash[60615]: ts=2024-07-28T19:36:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:39.109 INFO:tasks.workunit.client.0.smithi100.stdout:write: c 223966387~2981715 2024-07-28T19:36:39.138 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 196162870~5092767 2024-07-28T19:36:39.138 INFO:tasks.workunit.client.0.smithi100.stdout:write: c 51573393~3629098 2024-07-28T19:36:39.149 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 220301109~6646993 2024-07-28T19:36:39.170 INFO:tasks.workunit.client.0.smithi100.stdout:write: c 108815723~457851 2024-07-28T19:36:39.171 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 140052477~7106368 2024-07-28T19:36:39.172 INFO:tasks.workunit.client.0.smithi100.stdout:write: c 5918632~1996634 2024-07-28T19:36:39.173 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 170130690~124502 2024-07-28T19:36:39.173 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: user2 2024-07-28T19:36:39.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:39 smithi100 bash[80199]: cluster 2024-07-28T19:36:37.559222+0000 mgr.y (mgr.25358) 47 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s rd, 20 MiB/s wr, 565 op/s 2024-07-28T19:36:39.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:39 smithi100 bash[80199]: cluster 2024-07-28T19:36:38.083466+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-07-28T19:36:39.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:39 smithi100 bash[31715]: cluster 2024-07-28T19:36:37.559222+0000 mgr.y (mgr.25358) 47 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s rd, 20 MiB/s wr, 565 op/s 2024-07-28T19:36:39.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:39 smithi100 bash[31715]: cluster 2024-07-28T19:36:38.083466+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-07-28T19:36:39.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:39 smithi138 bash[23952]: cluster 2024-07-28T19:36:37.559222+0000 mgr.y (mgr.25358) 47 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 13 MiB/s rd, 20 MiB/s wr, 565 op/s 2024-07-28T19:36:39.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:39 smithi138 bash[23952]: cluster 2024-07-28T19:36:38.083466+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-07-28T19:36:40.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:40 smithi100 bash[80199]: cluster 2024-07-28T19:36:39.082155+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-07-28T19:36:40.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:40 smithi100 bash[31715]: cluster 2024-07-28T19:36:39.082155+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-07-28T19:36:40.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:40 smithi138 bash[23952]: cluster 2024-07-28T19:36:39.082155+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-07-28T19:36:41.126 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 160586393 2024-07-28T19:36:41.182 INFO:tasks.workunit.client.0.smithi100.stdout:write: d 46173059~3621953 2024-07-28T19:36:41.202 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 13431441~1223326 2024-07-28T19:36:41.203 INFO:tasks.workunit.client.0.smithi100.stdout:write: d 2793080~4139277 2024-07-28T19:36:41.214 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 26982766~5025412 2024-07-28T19:36:41.214 INFO:tasks.workunit.client.0.smithi100.stdout:write: d 153837579~4431628 2024-07-28T19:36:41.237 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 8796179~6283851 2024-07-28T19:36:41.243 INFO:tasks.workunit.client.0.smithi100.stdout:write: d 45163123~6848700 2024-07-28T19:36:41.258 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 94593427~6111529 2024-07-28T19:36:41.271 INFO:tasks.workunit.client.0.smithi100.stdout:snap_create: user3 2024-07-28T19:36:41.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:41 smithi100 bash[80199]: cluster 2024-07-28T19:36:39.560038+0000 mgr.y (mgr.25358) 48 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 38 MiB/s rd, 19 MiB/s wr, 647 op/s 2024-07-28T19:36:41.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:41 smithi100 bash[80199]: cluster 2024-07-28T19:36:40.107816+0000 mon.a (mon.0) 131 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-07-28T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:41 smithi100 bash[31715]: cluster 2024-07-28T19:36:39.560038+0000 mgr.y (mgr.25358) 48 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 38 MiB/s rd, 19 MiB/s wr, 647 op/s 2024-07-28T19:36:41.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:41 smithi100 bash[31715]: cluster 2024-07-28T19:36:40.107816+0000 mon.a (mon.0) 131 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-07-28T19:36:41.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:41 smithi138 bash[23952]: cluster 2024-07-28T19:36:39.560038+0000 mgr.y (mgr.25358) 48 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 38 MiB/s rd, 19 MiB/s wr, 647 op/s 2024-07-28T19:36:41.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:41 smithi138 bash[23952]: cluster 2024-07-28T19:36:40.107816+0000 mon.a (mon.0) 131 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-07-28T19:36:42.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:42 smithi100 bash[80199]: cluster 2024-07-28T19:36:41.109626+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-07-28T19:36:42.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:42 smithi100 bash[31715]: cluster 2024-07-28T19:36:41.109626+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-07-28T19:36:42.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:42 smithi138 bash[23952]: cluster 2024-07-28T19:36:41.109626+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-07-28T19:36:42.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:42 smithi138 bash[60615]: ts=2024-07-28T19:36:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:43.119 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:36:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:36:42] "GET /metrics HTTP/1.1" 200 38878 "" "Prometheus/2.43.0" 2024-07-28T19:36:43.157 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 190569332 2024-07-28T19:36:43.212 INFO:tasks.workunit.client.0.smithi100.stdout:write: 1 19056933~4764233 2024-07-28T19:36:43.216 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 20247991~2382116 2024-07-28T19:36:43.257 INFO:tasks.workunit.client.0.smithi100.stdout:write: 2 38113866~4764233 2024-07-28T19:36:43.260 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 39304924~2382116 2024-07-28T19:36:43.295 INFO:tasks.workunit.client.0.smithi100.stdout:write: 3 57170799~4764233 2024-07-28T19:36:43.297 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 58361857~2382116 2024-07-28T19:36:43.325 INFO:tasks.workunit.client.0.smithi100.stdout:write: 4 76227732~4764233 2024-07-28T19:36:43.327 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 77418790~2382116 2024-07-28T19:36:43.394 INFO:tasks.workunit.client.0.smithi100.stdout:write: 5 95284666~4764233 2024-07-28T19:36:43.397 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 96475724~2382116 2024-07-28T19:36:43.493 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:43 smithi100 bash[31715]: cluster 2024-07-28T19:36:41.560778+0000 mgr.y (mgr.25358) 49 : cluster [DBG] pgmap v87: 137 pgs: 137 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 38 MiB/s rd, 19 MiB/s wr, 647 op/s 2024-07-28T19:36:43.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:43 smithi100 bash[31715]: cluster 2024-07-28T19:36:42.130446+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-07-28T19:36:43.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:43 smithi100 bash[80199]: cluster 2024-07-28T19:36:41.560778+0000 mgr.y (mgr.25358) 49 : cluster [DBG] pgmap v87: 137 pgs: 137 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 38 MiB/s rd, 19 MiB/s wr, 647 op/s 2024-07-28T19:36:43.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:43 smithi100 bash[80199]: cluster 2024-07-28T19:36:42.130446+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-07-28T19:36:43.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:43 smithi138 bash[23952]: cluster 2024-07-28T19:36:41.560778+0000 mgr.y (mgr.25358) 49 : cluster [DBG] pgmap v87: 137 pgs: 137 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 38 MiB/s rd, 19 MiB/s wr, 647 op/s 2024-07-28T19:36:43.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:43 smithi138 bash[23952]: cluster 2024-07-28T19:36:42.130446+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-07-28T19:36:43.654 INFO:tasks.workunit.client.0.smithi100.stdout:migration_commit 2024-07-28T19:36:43.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:43 smithi138 bash[60615]: ts=2024-07-28T19:36:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:44.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:44 smithi138 bash[23952]: cluster 2024-07-28T19:36:43.141432+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-07-28T19:36:44.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:44 smithi100 bash[80199]: cluster 2024-07-28T19:36:43.141432+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-07-28T19:36:44.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:44 smithi100 bash[31715]: cluster 2024-07-28T19:36:43.141432+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-07-28T19:36:44.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:44 smithi138 bash[60615]: ts=2024-07-28T19:36:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:44.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:44 smithi138 bash[60615]: ts=2024-07-28T19:36:44.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:45.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:45 smithi100 bash[31715]: cluster 2024-07-28T19:36:43.562229+0000 mgr.y (mgr.25358) 50 : cluster [DBG] pgmap v90: 137 pgs: 137 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 5.5 MiB/s rd, 30 MiB/s wr, 424 op/s 2024-07-28T19:36:45.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:45 smithi100 bash[31715]: cluster 2024-07-28T19:36:44.154676+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-07-28T19:36:45.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:45 smithi100 bash[80199]: cluster 2024-07-28T19:36:43.562229+0000 mgr.y (mgr.25358) 50 : cluster [DBG] pgmap v90: 137 pgs: 137 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 5.5 MiB/s rd, 30 MiB/s wr, 424 op/s 2024-07-28T19:36:45.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:45 smithi100 bash[80199]: cluster 2024-07-28T19:36:44.154676+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-07-28T19:36:45.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:45 smithi138 bash[23952]: cluster 2024-07-28T19:36:43.562229+0000 mgr.y (mgr.25358) 50 : cluster [DBG] pgmap v90: 137 pgs: 137 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 5.5 MiB/s rd, 30 MiB/s wr, 424 op/s 2024-07-28T19:36:45.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:45 smithi138 bash[23952]: cluster 2024-07-28T19:36:44.154676+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-07-28T19:36:46.191 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMigration.StressLive (20760 ms) 2024-07-28T19:36:47.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:47 smithi100 bash[80199]: cluster 2024-07-28T19:36:45.563109+0000 mgr.y (mgr.25358) 51 : cluster [DBG] pgmap v92: 137 pgs: 137 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 5.0 MiB/s rd, 27 MiB/s wr, 380 op/s 2024-07-28T19:36:47.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:47 smithi100 bash[31715]: cluster 2024-07-28T19:36:45.563109+0000 mgr.y (mgr.25358) 51 : cluster [DBG] pgmap v92: 137 pgs: 137 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 5.0 MiB/s rd, 27 MiB/s wr, 380 op/s 2024-07-28T19:36:47.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:47 smithi138 bash[23952]: cluster 2024-07-28T19:36:45.563109+0000 mgr.y (mgr.25358) 51 : cluster [DBG] pgmap v92: 137 pgs: 137 active+clean; 2.3 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 5.0 MiB/s rd, 27 MiB/s wr, 380 op/s 2024-07-28T19:36:48.188 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 57 tests from TestMigration (289556 ms total) 2024-07-28T19:36:48.188 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:36:48.188 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 6 tests from TestGroup 2024-07-28T19:36:48.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:48 smithi100 bash[80199]: cluster 2024-07-28T19:36:47.163340+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-07-28T19:36:48.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:48 smithi100 bash[31715]: cluster 2024-07-28T19:36:47.163340+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-07-28T19:36:48.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:48 smithi138 bash[23952]: cluster 2024-07-28T19:36:47.163340+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-07-28T19:36:49.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:48 smithi138 bash[60615]: ts=2024-07-28T19:36:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:49.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:49 smithi100 bash[80199]: cluster 2024-07-28T19:36:47.563794+0000 mgr.y (mgr.25358) 52 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 4.4 MiB/s wr, 5 op/s 2024-07-28T19:36:49.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:49 smithi100 bash[80199]: cluster 2024-07-28T19:36:48.157128+0000 mon.a (mon.0) 137 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:36:49.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:49 smithi100 bash[80199]: cluster 2024-07-28T19:36:48.173694+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-07-28T19:36:49.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:49 smithi100 bash[31715]: cluster 2024-07-28T19:36:47.563794+0000 mgr.y (mgr.25358) 52 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 4.4 MiB/s wr, 5 op/s 2024-07-28T19:36:49.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:49 smithi100 bash[31715]: cluster 2024-07-28T19:36:48.157128+0000 mon.a (mon.0) 137 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:36:49.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:49 smithi100 bash[31715]: cluster 2024-07-28T19:36:48.173694+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-07-28T19:36:49.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:49 smithi138 bash[23952]: cluster 2024-07-28T19:36:47.563794+0000 mgr.y (mgr.25358) 52 : cluster [DBG] pgmap v94: 105 pgs: 105 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 4.4 MiB/s wr, 5 op/s 2024-07-28T19:36:49.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:49 smithi138 bash[23952]: cluster 2024-07-28T19:36:48.157128+0000 mon.a (mon.0) 137 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:36:49.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:49 smithi138 bash[23952]: cluster 2024-07-28T19:36:48.173694+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-07-28T19:36:50.180 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestGroup.group_create 2024-07-28T19:36:50.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[80199]: cluster 2024-07-28T19:36:49.175465+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-07-28T19:36:50.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[80199]: audit 2024-07-28T19:36:49.185426+0000 mon.c (mon.2) 205 : audit [INF] from='client.? 172.21.15.100:0/1058766080' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:36:50.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[80199]: audit 2024-07-28T19:36:49.198265+0000 mon.a (mon.0) 140 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:36:50.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[80199]: audit 2024-07-28T19:36:50.181441+0000 mon.a (mon.0) 141 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:36:50.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[80199]: cluster 2024-07-28T19:36:50.181977+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-07-28T19:36:50.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[31715]: cluster 2024-07-28T19:36:49.175465+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-07-28T19:36:50.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[31715]: audit 2024-07-28T19:36:49.185426+0000 mon.c (mon.2) 205 : audit [INF] from='client.? 172.21.15.100:0/1058766080' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:36:50.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[31715]: audit 2024-07-28T19:36:49.198265+0000 mon.a (mon.0) 140 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:36:50.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[31715]: audit 2024-07-28T19:36:50.181441+0000 mon.a (mon.0) 141 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:36:50.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:50 smithi100 bash[31715]: cluster 2024-07-28T19:36:50.181977+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-07-28T19:36:50.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:50 smithi138 bash[23952]: cluster 2024-07-28T19:36:49.175465+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-07-28T19:36:50.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:50 smithi138 bash[23952]: audit 2024-07-28T19:36:49.185426+0000 mon.c (mon.2) 205 : audit [INF] from='client.? 172.21.15.100:0/1058766080' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:36:50.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:50 smithi138 bash[23952]: audit 2024-07-28T19:36:49.198265+0000 mon.a (mon.0) 140 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:36:50.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:50 smithi138 bash[23952]: audit 2024-07-28T19:36:50.181441+0000 mon.a (mon.0) 141 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:36:50.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:50 smithi138 bash[23952]: cluster 2024-07-28T19:36:50.181977+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-07-28T19:36:51.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:51 smithi100 bash[80199]: cluster 2024-07-28T19:36:49.564444+0000 mgr.y (mgr.25358) 53 : cluster [DBG] pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-07-28T19:36:51.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:51 smithi100 bash[80199]: cluster 2024-07-28T19:36:50.185015+0000 mon.a (mon.0) 143 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:36:51.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:51 smithi100 bash[31715]: cluster 2024-07-28T19:36:49.564444+0000 mgr.y (mgr.25358) 53 : cluster [DBG] pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-07-28T19:36:51.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:51 smithi100 bash[31715]: cluster 2024-07-28T19:36:50.185015+0000 mon.a (mon.0) 143 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:36:51.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:51 smithi138 bash[23952]: cluster 2024-07-28T19:36:49.564444+0000 mgr.y (mgr.25358) 53 : cluster [DBG] pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-07-28T19:36:51.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:51 smithi138 bash[23952]: cluster 2024-07-28T19:36:50.185015+0000 mon.a (mon.0) 143 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:36:52.288 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestGroup.group_create (2107 ms) 2024-07-28T19:36:52.288 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestGroup.group_createPP 2024-07-28T19:36:52.333 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestGroup.group_createPP (46 ms) 2024-07-28T19:36:52.334 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestGroup.add_image 2024-07-28T19:36:52.432 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestGroup.add_image (99 ms) 2024-07-28T19:36:52.433 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestGroup.add_imagePP 2024-07-28T19:36:52.493 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:52 smithi100 bash[80199]: cluster 2024-07-28T19:36:51.225548+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-07-28T19:36:52.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:52 smithi100 bash[31715]: cluster 2024-07-28T19:36:51.225548+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-07-28T19:36:52.515 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestGroup.add_imagePP (83 ms) 2024-07-28T19:36:52.515 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestGroup.add_snapshot 2024-07-28T19:36:52.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:52 smithi138 bash[23952]: cluster 2024-07-28T19:36:51.225548+0000 mon.a (mon.0) 144 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-07-28T19:36:52.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:52 smithi138 bash[60615]: ts=2024-07-28T19:36:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:53.227 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:36:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:36:52] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2024-07-28T19:36:53.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[31715]: cluster 2024-07-28T19:36:51.565132+0000 mgr.y (mgr.25358) 54 : cluster [DBG] pgmap v100: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-07-28T19:36:53.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[31715]: cluster 2024-07-28T19:36:52.220236+0000 mon.a (mon.0) 145 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:36:53.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[31715]: cluster 2024-07-28T19:36:52.242639+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-07-28T19:36:53.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[31715]: audit 2024-07-28T19:36:52.632791+0000 mon.a (mon.0) 147 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:53.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[31715]: audit 2024-07-28T19:36:52.634334+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[80199]: cluster 2024-07-28T19:36:51.565132+0000 mgr.y (mgr.25358) 54 : cluster [DBG] pgmap v100: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-07-28T19:36:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[80199]: cluster 2024-07-28T19:36:52.220236+0000 mon.a (mon.0) 145 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:36:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[80199]: cluster 2024-07-28T19:36:52.242639+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-07-28T19:36:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[80199]: audit 2024-07-28T19:36:52.632791+0000 mon.a (mon.0) 147 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:53.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:53 smithi100 bash[80199]: audit 2024-07-28T19:36:52.634334+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:53.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:53 smithi138 bash[23952]: cluster 2024-07-28T19:36:51.565132+0000 mgr.y (mgr.25358) 54 : cluster [DBG] pgmap v100: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-07-28T19:36:53.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:53 smithi138 bash[23952]: cluster 2024-07-28T19:36:52.220236+0000 mon.a (mon.0) 145 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:36:53.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:53 smithi138 bash[23952]: cluster 2024-07-28T19:36:52.242639+0000 mon.a (mon.0) 146 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-07-28T19:36:53.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:53 smithi138 bash[23952]: audit 2024-07-28T19:36:52.632791+0000 mon.a (mon.0) 147 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:53.504 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:53 smithi138 bash[23952]: audit 2024-07-28T19:36:52.634334+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:36:53.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:53 smithi138 bash[60615]: ts=2024-07-28T19:36:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:54.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:54 smithi138 bash[23952]: cluster 2024-07-28T19:36:53.566319+0000 mgr.y (mgr.25358) 55 : cluster [DBG] pgmap v102: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 12 KiB/s wr, 97 op/s 2024-07-28T19:36:54.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:54 smithi138 bash[23952]: cluster 2024-07-28T19:36:53.637362+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-07-28T19:36:54.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:54 smithi138 bash[60615]: ts=2024-07-28T19:36:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:54.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:54 smithi138 bash[60615]: ts=2024-07-28T19:36:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:54.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:54 smithi100 bash[80199]: cluster 2024-07-28T19:36:53.566319+0000 mgr.y (mgr.25358) 55 : cluster [DBG] pgmap v102: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 12 KiB/s wr, 97 op/s 2024-07-28T19:36:54.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:54 smithi100 bash[80199]: cluster 2024-07-28T19:36:53.637362+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-07-28T19:36:54.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:54 smithi100 bash[31715]: cluster 2024-07-28T19:36:53.566319+0000 mgr.y (mgr.25358) 55 : cluster [DBG] pgmap v102: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 12 KiB/s wr, 97 op/s 2024-07-28T19:36:54.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:54 smithi100 bash[31715]: cluster 2024-07-28T19:36:53.637362+0000 mon.a (mon.0) 149 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-07-28T19:36:55.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:55 smithi100 bash[31715]: cluster 2024-07-28T19:36:54.652576+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-07-28T19:36:55.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:55 smithi100 bash[31715]: cluster 2024-07-28T19:36:54.790305+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-07-28T19:36:55.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:55 smithi100 bash[31715]: audit 2024-07-28T19:36:55.497687+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:36:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:55 smithi100 bash[80199]: cluster 2024-07-28T19:36:54.652576+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-07-28T19:36:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:55 smithi100 bash[80199]: cluster 2024-07-28T19:36:54.790305+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-07-28T19:36:55.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:55 smithi100 bash[80199]: audit 2024-07-28T19:36:55.497687+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:36:56.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:55 smithi138 bash[23952]: cluster 2024-07-28T19:36:54.652576+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-07-28T19:36:56.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:55 smithi138 bash[23952]: cluster 2024-07-28T19:36:54.790305+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-07-28T19:36:56.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:55 smithi138 bash[23952]: audit 2024-07-28T19:36:55.497687+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:36:57.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:56 smithi138 bash[23952]: cluster 2024-07-28T19:36:55.566783+0000 mgr.y (mgr.25358) 56 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-07-28T19:36:57.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:56 smithi138 bash[23952]: cluster 2024-07-28T19:36:55.797756+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-07-28T19:36:57.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:56 smithi138 bash[23952]: audit 2024-07-28T19:36:55.874195+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:57.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:56 smithi138 bash[23952]: audit 2024-07-28T19:36:55.879530+0000 mon.a (mon.0) 155 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:56 smithi100 bash[31715]: cluster 2024-07-28T19:36:55.566783+0000 mgr.y (mgr.25358) 56 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-07-28T19:36:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:56 smithi100 bash[31715]: cluster 2024-07-28T19:36:55.797756+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-07-28T19:36:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:56 smithi100 bash[31715]: audit 2024-07-28T19:36:55.874195+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:57.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:56 smithi100 bash[31715]: audit 2024-07-28T19:36:55.879530+0000 mon.a (mon.0) 155 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:57.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:56 smithi100 bash[80199]: cluster 2024-07-28T19:36:55.566783+0000 mgr.y (mgr.25358) 56 : cluster [DBG] pgmap v106: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-07-28T19:36:57.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:56 smithi100 bash[80199]: cluster 2024-07-28T19:36:55.797756+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-07-28T19:36:57.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:56 smithi100 bash[80199]: audit 2024-07-28T19:36:55.874195+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:57.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:56 smithi100 bash[80199]: audit 2024-07-28T19:36:55.879530+0000 mon.a (mon.0) 155 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:36:57.865 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestGroup.add_snapshot (5350 ms) 2024-07-28T19:36:57.866 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-07-28T19:36:58.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:57 smithi100 bash[80199]: cluster 2024-07-28T19:36:56.798604+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-07-28T19:36:58.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:57 smithi100 bash[31715]: cluster 2024-07-28T19:36:56.798604+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-07-28T19:36:58.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:57 smithi138 bash[23952]: cluster 2024-07-28T19:36:56.798604+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-07-28T19:36:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:58 smithi138 bash[23952]: cluster 2024-07-28T19:36:57.567415+0000 mgr.y (mgr.25358) 57 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:36:59.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:58 smithi138 bash[23952]: cluster 2024-07-28T19:36:57.821985+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-07-28T19:36:59.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:36:58 smithi138 bash[60615]: ts=2024-07-28T19:36:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:36:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:58 smithi100 bash[80199]: cluster 2024-07-28T19:36:57.567415+0000 mgr.y (mgr.25358) 57 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:36:59.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:58 smithi100 bash[80199]: cluster 2024-07-28T19:36:57.821985+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-07-28T19:36:59.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:58 smithi100 bash[31715]: cluster 2024-07-28T19:36:57.567415+0000 mgr.y (mgr.25358) 57 : cluster [DBG] pgmap v109: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:36:59.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:58 smithi100 bash[31715]: cluster 2024-07-28T19:36:57.821985+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-07-28T19:36:59.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:36:59 smithi100 bash[80199]: cluster 2024-07-28T19:36:58.819662+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-07-28T19:36:59.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:36:59 smithi100 bash[31715]: cluster 2024-07-28T19:36:58.819662+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-07-28T19:37:00.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:36:59 smithi138 bash[23952]: cluster 2024-07-28T19:36:58.819662+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-07-28T19:37:00.520 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch upgrade resume' 2024-07-28T19:37:01.243 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:00 smithi100 bash[80199]: cluster 2024-07-28T19:36:59.568145+0000 mgr.y (mgr.25358) 58 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 170 op/s 2024-07-28T19:37:01.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:00 smithi100 bash[80199]: cluster 2024-07-28T19:36:59.846273+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-07-28T19:37:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:00 smithi100 bash[31715]: cluster 2024-07-28T19:36:59.568145+0000 mgr.y (mgr.25358) 58 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 170 op/s 2024-07-28T19:37:01.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:00 smithi100 bash[31715]: cluster 2024-07-28T19:36:59.846273+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-07-28T19:37:01.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:00 smithi138 bash[23952]: cluster 2024-07-28T19:36:59.568145+0000 mgr.y (mgr.25358) 58 : cluster [DBG] pgmap v112: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 170 op/s 2024-07-28T19:37:01.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:00 smithi138 bash[23952]: cluster 2024-07-28T19:36:59.846273+0000 mon.a (mon.0) 159 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-07-28T19:37:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[80199]: cluster 2024-07-28T19:37:00.850643+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-07-28T19:37:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[80199]: audit 2024-07-28T19:37:01.352952+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[80199]: audit 2024-07-28T19:37:01.359487+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[80199]: audit 2024-07-28T19:37:01.666898+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[80199]: audit 2024-07-28T19:37:01.672189+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[31715]: cluster 2024-07-28T19:37:00.850643+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-07-28T19:37:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[31715]: audit 2024-07-28T19:37:01.352952+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[31715]: audit 2024-07-28T19:37:01.359487+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[31715]: audit 2024-07-28T19:37:01.666898+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:01 smithi100 bash[31715]: audit 2024-07-28T19:37:01.672189+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:01 smithi138 bash[23952]: cluster 2024-07-28T19:37:00.850643+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-07-28T19:37:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:01 smithi138 bash[23952]: audit 2024-07-28T19:37:01.352952+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:01 smithi138 bash[23952]: audit 2024-07-28T19:37:01.359487+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:01 smithi138 bash[23952]: audit 2024-07-28T19:37:01.666898+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:01 smithi138 bash[23952]: audit 2024-07-28T19:37:01.672189+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:02.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:02 smithi138 bash[60615]: ts=2024-07-28T19:37:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:03.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:37:02] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2024-07-28T19:37:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[80199]: cluster 2024-07-28T19:37:01.568776+0000 mgr.y (mgr.25358) 59 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 170 op/s 2024-07-28T19:37:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[80199]: cluster 2024-07-28T19:37:01.852298+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-07-28T19:37:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[80199]: audit 2024-07-28T19:37:02.066407+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[80199]: audit 2024-07-28T19:37:02.067946+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:03.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[80199]: audit 2024-07-28T19:37:02.077206+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:03.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[31715]: cluster 2024-07-28T19:37:01.568776+0000 mgr.y (mgr.25358) 59 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 170 op/s 2024-07-28T19:37:03.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[31715]: cluster 2024-07-28T19:37:01.852298+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-07-28T19:37:03.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[31715]: audit 2024-07-28T19:37:02.066407+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:03.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[31715]: audit 2024-07-28T19:37:02.067946+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:03.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:02 smithi100 bash[31715]: audit 2024-07-28T19:37:02.077206+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:03.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:02 smithi138 bash[23952]: cluster 2024-07-28T19:37:01.568776+0000 mgr.y (mgr.25358) 59 : cluster [DBG] pgmap v115: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 170 op/s 2024-07-28T19:37:03.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:02 smithi138 bash[23952]: cluster 2024-07-28T19:37:01.852298+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-07-28T19:37:03.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:02 smithi138 bash[23952]: audit 2024-07-28T19:37:02.066407+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:03.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:03 smithi138 bash[23952]: audit 2024-07-28T19:37:02.067946+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:03.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:03 smithi138 bash[23952]: audit 2024-07-28T19:37:02.077206+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:03.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:03 smithi138 bash[60615]: ts=2024-07-28T19:37:03.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:03.920 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestGroup.add_snapshotPP (6054 ms) 2024-07-28T19:37:04.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:03 smithi100 bash[80199]: cluster 2024-07-28T19:37:02.871997+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-07-28T19:37:04.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:03 smithi100 bash[31715]: cluster 2024-07-28T19:37:02.871997+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-07-28T19:37:04.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:03 smithi138 bash[23952]: cluster 2024-07-28T19:37:02.871997+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-07-28T19:37:04.797 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 6 tests from TestGroup (13739 ms total) 2024-07-28T19:37:04.797 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:37:04.797 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 31 tests from TestDeepCopy 2024-07-28T19:37:04.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:04 smithi138 bash[60615]: ts=2024-07-28T19:37:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:04.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:04 smithi138 bash[60615]: ts=2024-07-28T19:37:04.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:05.070 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:37:05.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:05 smithi100 bash[80199]: cluster 2024-07-28T19:37:03.570039+0000 mgr.y (mgr.25358) 60 : cluster [DBG] pgmap v118: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 95 KiB/s rd, 7.7 KiB/s wr, 124 op/s 2024-07-28T19:37:05.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:05 smithi100 bash[80199]: cluster 2024-07-28T19:37:03.882691+0000 mon.a (mon.0) 170 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-07-28T19:37:05.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:05 smithi100 bash[80199]: cluster 2024-07-28T19:37:04.791980+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-07-28T19:37:05.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:05 smithi100 bash[31715]: cluster 2024-07-28T19:37:03.570039+0000 mgr.y (mgr.25358) 60 : cluster [DBG] pgmap v118: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 95 KiB/s rd, 7.7 KiB/s wr, 124 op/s 2024-07-28T19:37:05.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:05 smithi100 bash[31715]: cluster 2024-07-28T19:37:03.882691+0000 mon.a (mon.0) 170 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-07-28T19:37:05.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:05 smithi100 bash[31715]: cluster 2024-07-28T19:37:04.791980+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-07-28T19:37:05.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:05 smithi138 bash[23952]: cluster 2024-07-28T19:37:03.570039+0000 mgr.y (mgr.25358) 60 : cluster [DBG] pgmap v118: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 95 KiB/s rd, 7.7 KiB/s wr, 124 op/s 2024-07-28T19:37:05.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:05 smithi138 bash[23952]: cluster 2024-07-28T19:37:03.882691+0000 mon.a (mon.0) 170 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-07-28T19:37:05.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:05 smithi138 bash[23952]: cluster 2024-07-28T19:37:04.791980+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-07-28T19:37:06.811 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Empty 2024-07-28T19:37:06.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:06 smithi100 bash[31715]: cluster 2024-07-28T19:37:05.570641+0000 mgr.y (mgr.25358) 61 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:06.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:06 smithi100 bash[31715]: cluster 2024-07-28T19:37:05.798067+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-07-28T19:37:06.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:06 smithi100 bash[31715]: audit 2024-07-28T19:37:05.802942+0000 mon.c (mon.2) 206 : audit [INF] from='client.? 172.21.15.100:0/1203007186' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:37:06.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:06 smithi100 bash[31715]: audit 2024-07-28T19:37:05.803777+0000 mon.a (mon.0) 173 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:37:06.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:06 smithi100 bash[80199]: cluster 2024-07-28T19:37:05.570641+0000 mgr.y (mgr.25358) 61 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:06.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:06 smithi100 bash[80199]: cluster 2024-07-28T19:37:05.798067+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-07-28T19:37:06.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:06 smithi100 bash[80199]: audit 2024-07-28T19:37:05.802942+0000 mon.c (mon.2) 206 : audit [INF] from='client.? 172.21.15.100:0/1203007186' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:37:06.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:06 smithi100 bash[80199]: audit 2024-07-28T19:37:05.803777+0000 mon.a (mon.0) 173 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:37:07.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:06 smithi138 bash[23952]: cluster 2024-07-28T19:37:05.570641+0000 mgr.y (mgr.25358) 61 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:07.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:06 smithi138 bash[23952]: cluster 2024-07-28T19:37:05.798067+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-07-28T19:37:07.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:06 smithi138 bash[23952]: audit 2024-07-28T19:37:05.802942+0000 mon.c (mon.2) 206 : audit [INF] from='client.? 172.21.15.100:0/1203007186' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:37:07.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:06 smithi138 bash[23952]: audit 2024-07-28T19:37:05.803777+0000 mon.a (mon.0) 173 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:37:08.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:07 smithi100 bash[80199]: audit 2024-07-28T19:37:06.800409+0000 mon.a (mon.0) 174 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:37:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:07 smithi100 bash[80199]: cluster 2024-07-28T19:37:06.800835+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-07-28T19:37:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:07 smithi100 bash[80199]: audit 2024-07-28T19:37:07.626595+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:08.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:07 smithi100 bash[31715]: audit 2024-07-28T19:37:06.800409+0000 mon.a (mon.0) 174 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:37:08.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:07 smithi100 bash[31715]: cluster 2024-07-28T19:37:06.800835+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-07-28T19:37:08.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:07 smithi100 bash[31715]: audit 2024-07-28T19:37:07.626595+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:08.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:07 smithi138 bash[23952]: audit 2024-07-28T19:37:06.800409+0000 mon.a (mon.0) 174 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:37:08.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:07 smithi138 bash[23952]: cluster 2024-07-28T19:37:06.800835+0000 mon.a (mon.0) 175 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-07-28T19:37:08.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:07 smithi138 bash[23952]: audit 2024-07-28T19:37:07.626595+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:08.894 INFO:teuthology.orchestra.run.smithi100.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:37:09.007 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Empty (2197 ms) 2024-07-28T19:37:09.007 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-07-28T19:37:09.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:08 smithi138 bash[60615]: ts=2024-07-28T19:37:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:09.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:08 smithi138 bash[23952]: cluster 2024-07-28T19:37:07.571327+0000 mgr.y (mgr.25358) 62 : cluster [DBG] pgmap v124: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:09.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:08 smithi138 bash[23952]: cluster 2024-07-28T19:37:07.827097+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-07-28T19:37:09.097 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:08 smithi100 bash[80199]: cluster 2024-07-28T19:37:07.571327+0000 mgr.y (mgr.25358) 62 : cluster [DBG] pgmap v124: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:09.098 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:08 smithi100 bash[80199]: cluster 2024-07-28T19:37:07.827097+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-07-28T19:37:09.098 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:08 smithi100 bash[31715]: cluster 2024-07-28T19:37:07.571327+0000 mgr.y (mgr.25358) 62 : cluster [DBG] pgmap v124: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:09.098 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:08 smithi100 bash[31715]: cluster 2024-07-28T19:37:07.827097+0000 mon.a (mon.0) 177 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-07-28T19:37:09.341 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.NoSnaps (334 ms) 2024-07-28T19:37:09.342 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Snaps 2024-07-28T19:37:09.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: cluster 2024-07-28T19:37:08.845958+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-07-28T19:37:09.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: cluster 2024-07-28T19:37:08.847153+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-07-28T19:37:09.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:08.894009+0000 mgr.y (mgr.25358) 63 : audit [DBG] from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:09.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: cephadm 2024-07-28T19:37:08.894519+0000 mgr.y (mgr.25358) 64 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:37:09.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:08.899722+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:08.900806+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:08.903636+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:08.904507+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:08.910443+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:09.000078+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:09.002959+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:09.004398+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: audit 2024-07-28T19:37:09.005301+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[80199]: cephadm 2024-07-28T19:37:09.005888+0000 mgr.y (mgr.25358) 65 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: cluster 2024-07-28T19:37:08.845958+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-07-28T19:37:09.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: cluster 2024-07-28T19:37:08.847153+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:08.894009+0000 mgr.y (mgr.25358) 63 : audit [DBG] from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: cephadm 2024-07-28T19:37:08.894519+0000 mgr.y (mgr.25358) 64 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:08.899722+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:08.900806+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:08.903636+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:08.904507+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:08.910443+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:09.000078+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:09.002959+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:09.004398+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: audit 2024-07-28T19:37:09.005301+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-07-28T19:37:09.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:09 smithi100 bash[31715]: cephadm 2024-07-28T19:37:09.005888+0000 mgr.y (mgr.25358) 65 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-07-28T19:37:10.022 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'echo wait for majority of mons to upgrade' 2024-07-28T19:37:10.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: cluster 2024-07-28T19:37:08.845958+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-07-28T19:37:10.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: cluster 2024-07-28T19:37:08.847153+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-07-28T19:37:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:08.894009+0000 mgr.y (mgr.25358) 63 : audit [DBG] from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: cephadm 2024-07-28T19:37:08.894519+0000 mgr.y (mgr.25358) 64 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:37:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:08.899722+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:08.900806+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:08.903636+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:08.904507+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:08.910443+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:09.000078+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:10.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:09.002959+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:37:10.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:09.004398+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:37:10.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: audit 2024-07-28T19:37:09.005301+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-07-28T19:37:10.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:09 smithi138 bash[23952]: cephadm 2024-07-28T19:37:09.005888+0000 mgr.y (mgr.25358) 65 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-07-28T19:37:11.157 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:10 smithi100 bash[80199]: cluster 2024-07-28T19:37:09.571996+0000 mgr.y (mgr.25358) 66 : cluster [DBG] pgmap v127: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2024-07-28T19:37:11.157 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:10 smithi100 bash[80199]: cluster 2024-07-28T19:37:09.921895+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-07-28T19:37:11.157 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:10 smithi100 bash[31715]: cluster 2024-07-28T19:37:09.571996+0000 mgr.y (mgr.25358) 66 : cluster [DBG] pgmap v127: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2024-07-28T19:37:11.157 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:10 smithi100 bash[31715]: cluster 2024-07-28T19:37:09.921895+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-07-28T19:37:11.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:10 smithi138 bash[23952]: cluster 2024-07-28T19:37:09.571996+0000 mgr.y (mgr.25358) 66 : cluster [DBG] pgmap v127: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2024-07-28T19:37:11.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:10 smithi138 bash[23952]: cluster 2024-07-28T19:37:09.921895+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-07-28T19:37:12.278 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[80199]: cluster 2024-07-28T19:37:10.927029+0000 mon.a (mon.0) 190 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-07-28T19:37:12.278 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[80199]: audit 2024-07-28T19:37:11.755568+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:12.278 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[80199]: audit 2024-07-28T19:37:11.756740+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:37:12.278 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[80199]: audit 2024-07-28T19:37:11.757595+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:37:12.278 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[80199]: audit 2024-07-28T19:37:11.758436+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:12.278 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[31715]: cluster 2024-07-28T19:37:10.927029+0000 mon.a (mon.0) 190 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-07-28T19:37:12.279 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[31715]: audit 2024-07-28T19:37:11.755568+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:12.279 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[31715]: audit 2024-07-28T19:37:11.756740+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:37:12.279 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[31715]: audit 2024-07-28T19:37:11.757595+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:37:12.279 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:11 smithi100 bash[31715]: audit 2024-07-28T19:37:11.758436+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:12.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:11 smithi138 bash[23952]: cluster 2024-07-28T19:37:10.927029+0000 mon.a (mon.0) 190 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-07-28T19:37:12.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:11 smithi138 bash[23952]: audit 2024-07-28T19:37:11.755568+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:12.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:11 smithi138 bash[23952]: audit 2024-07-28T19:37:11.756740+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:37:12.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:11 smithi138 bash[23952]: audit 2024-07-28T19:37:11.757595+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:37:12.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:11 smithi138 bash[23952]: audit 2024-07-28T19:37:11.758436+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:12.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:12 smithi138 bash[60615]: ts=2024-07-28T19:37:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:13.232 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:37:12] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.43.0" 2024-07-28T19:37:13.232 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[80199]: cluster 2024-07-28T19:37:11.572662+0000 mgr.y (mgr.25358) 67 : cluster [DBG] pgmap v130: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2024-07-28T19:37:13.232 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[80199]: cephadm 2024-07-28T19:37:11.750330+0000 mgr.y (mgr.25358) 68 : cephadm [INF] Upgrade: Updating mon.c 2024-07-28T19:37:13.232 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[80199]: cephadm 2024-07-28T19:37:11.759496+0000 mgr.y (mgr.25358) 69 : cephadm [INF] Deploying daemon mon.c on smithi100 2024-07-28T19:37:13.232 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[80199]: cluster 2024-07-28T19:37:11.939551+0000 mon.a (mon.0) 195 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-07-28T19:37:13.233 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[31715]: cluster 2024-07-28T19:37:11.572662+0000 mgr.y (mgr.25358) 67 : cluster [DBG] pgmap v130: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2024-07-28T19:37:13.233 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[31715]: cephadm 2024-07-28T19:37:11.750330+0000 mgr.y (mgr.25358) 68 : cephadm [INF] Upgrade: Updating mon.c 2024-07-28T19:37:13.233 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[31715]: cephadm 2024-07-28T19:37:11.759496+0000 mgr.y (mgr.25358) 69 : cephadm [INF] Deploying daemon mon.c on smithi100 2024-07-28T19:37:13.233 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:12 smithi100 bash[31715]: cluster 2024-07-28T19:37:11.939551+0000 mon.a (mon.0) 195 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-07-28T19:37:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:12 smithi138 bash[23952]: cluster 2024-07-28T19:37:11.572662+0000 mgr.y (mgr.25358) 67 : cluster [DBG] pgmap v130: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s wr, 0 op/s 2024-07-28T19:37:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:12 smithi138 bash[23952]: cephadm 2024-07-28T19:37:11.750330+0000 mgr.y (mgr.25358) 68 : cephadm [INF] Upgrade: Updating mon.c 2024-07-28T19:37:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:12 smithi138 bash[23952]: cephadm 2024-07-28T19:37:11.759496+0000 mgr.y (mgr.25358) 69 : cephadm [INF] Deploying daemon mon.c on smithi100 2024-07-28T19:37:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:12 smithi138 bash[23952]: cluster 2024-07-28T19:37:11.939551+0000 mon.a (mon.0) 195 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-07-28T19:37:13.342 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Snaps (4000 ms) 2024-07-28T19:37:13.342 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-07-28T19:37:13.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:13 smithi138 bash[60615]: ts=2024-07-28T19:37:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:14.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:13 smithi100 bash[31715]: cluster 2024-07-28T19:37:12.947724+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-07-28T19:37:14.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:13 smithi100 bash[80199]: cluster 2024-07-28T19:37:12.947724+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-07-28T19:37:14.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:13 smithi138 bash[23952]: cluster 2024-07-28T19:37:12.947724+0000 mon.a (mon.0) 196 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-07-28T19:37:14.666 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.666 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.667 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.667 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.667 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.668 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.668 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.668 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: Stopping Ceph mon.c for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:37:14.668 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:14 smithi100 bash[31715]: debug 2024-07-28T19:37:14.550+0000 7f815165d640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-28T19:37:14.668 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:14 smithi100 bash[31715]: debug 2024-07-28T19:37:14.550+0000 7f815165d640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-07-28T19:37:14.668 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.669 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:37:14 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:14.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:14 smithi138 bash[60615]: ts=2024-07-28T19:37:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:14.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:14 smithi138 bash[60615]: ts=2024-07-28T19:37:14.458Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:15.590 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:15 smithi100 bash[82527]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-c 2024-07-28T19:37:15.590 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:15 smithi100 bash[82590]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-c 2024-07-28T19:37:15.590 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.c.service: Deactivated successfully. 2024-07-28T19:37:15.590 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: Stopped Ceph mon.c for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:37:15.957 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:37:15.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:15.995 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:15.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:15.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: Started Ceph mon.c for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:37:15.995 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:15.995 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:15.996 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:15.996 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:15.996 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:15.997 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:37:15 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:17.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.690+0000 7efd455b1b00 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.690+0000 7efd455b1b00 0 ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable), process ceph-mon, pid 7 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.690+0000 7efd455b1b00 0 pidfile_write: ignore empty --pid-file 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 0 load: jerasure load: lrc 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: RocksDB version: 7.9.2 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Git sha 0 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Compile date 2024-07-22 20:55:56 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: DB SUMMARY 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: DB Session ID: SMGEQQAIB6DT6Y2QQ2FS 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: CURRENT file: CURRENT 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: IDENTITY file: IDENTITY 2024-07-28T19:37:17.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 1846 Bytes 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000039.sst 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000037.log size: 4142355 ; 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.error_if_exists: 0 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.create_if_missing: 0 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.paranoid_checks: 1 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.env: 0x561b1a42abc0 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.fs: PosixFileSystem 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.info_log: 0x561b1b3bebe0 2024-07-28T19:37:17.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.statistics: (nil) 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.use_fsync: 0 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_log_file_size: 0 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:17.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:17.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:17.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:17.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:17.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[80199]: debug 2024-07-28T19:37:17.834+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:17.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.allow_fallocate: 1 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.use_direct_reads: 0 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.create_missing_column_families: 0 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.db_log_dir: 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.wal_dir: 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-28T19:37:18.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.advise_random_on_open: 1 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.write_buffer_manager: 0x561b1b3ccb40 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.rate_limiter: (nil) 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-28T19:37:18.001 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.unordered_write: 0 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.row_cache: None 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.wal_filter: None 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.two_write_queues: 0 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.manual_wal_flush: 0 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.wal_compression: 0 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.atomic_flush: 0 2024-07-28T19:37:18.002 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.log_readahead_size: 0 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_background_jobs: 2 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_background_compactions: -1 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_subcompactions: 1 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-28T19:37:18.003 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_total_wal_size: 0 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_open_files: -1 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bytes_per_sync: 0 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_background_flushes: -1 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Compression algorithms supported: 2024-07-28T19:37:18.004 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: kZSTD supported: 0 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: kXpressCompression supported: 0 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: kBZip2Compression supported: 0 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: kLZ4Compression supported: 1 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: kZlibCompression supported: 1 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: kSnappyCompression supported: 1 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.merge_operator: 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_filter: None 2024-07-28T19:37:18.005 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_filter_factory: None 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561b1b3be820) 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: cache_index_and_filter_blocks: 1 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: pin_top_level_index_and_filter: 1 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: index_type: 0 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: data_block_index_type: 0 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: index_shortening: 1 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: data_block_hash_table_util_ratio: 0.750000 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: checksum: 4 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: no_block_cache: 0 2024-07-28T19:37:18.006 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: block_cache: 0x561b1b3b31f0 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: block_cache_name: BinnedLRUCache 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: block_cache_options: 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: capacity : 536870912 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: num_shard_bits : 4 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: strict_capacity_limit : 0 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: high_pri_pool_ratio: 0.000 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: block_cache_compressed: (nil) 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: persistent_cache: (nil) 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: block_size: 4096 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: block_size_deviation: 10 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: block_restart_interval: 16 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: index_block_restart_interval: 1 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: metadata_block_size: 4096 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: partition_filters: 0 2024-07-28T19:37:18.007 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: use_delta_encoding: 1 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: filter_policy: bloomfilter 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: whole_key_filtering: 1 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: verify_compression: 0 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: read_amp_bytes_per_bit: 0 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: format_version: 5 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: enable_index_compression: 1 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: block_align: 0 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: max_auto_readahead_size: 262144 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: prepopulate_block_cache: 0 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: initial_auto_readahead_size: 8192 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: num_file_reads_for_auto_readahead: 2 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression: NoCompression 2024-07-28T19:37:18.008 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.num_levels: 7 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:37:18.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.level: 32767 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.enabled: false 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-28T19:37:18.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-28T19:37:18.011 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.arena_block_size: 1048576 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-28T19:37:18.012 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.table_properties_collectors: 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.inplace_update_support: 0 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.bloom_locality: 0 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.max_successive_merges: 0 2024-07-28T19:37:18.013 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.force_consistency_checks: 1 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.ttl: 2592000 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.enable_blob_files: false 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.min_blob_size: 0 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.blob_file_size: 268435456 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-28T19:37:18.014 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.blob_file_starting_level: 0 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.694+0000 7efd455b1b00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.698+0000 7efd4238a640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 39.sst 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.698+0000 7efd455b1b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.698+0000 7efd455b1b00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 41, last_sequence is 21498, log_number is 37,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.698+0000 7efd455b1b00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 37 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.698+0000 7efd455b1b00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8048c795-9f83-4f2e-a5ed-755167692c00 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.698+0000 7efd455b1b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195437701929, "job": 1, "event": "recovery_started", "wal_files": [37]} 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.698+0000 7efd455b1b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #37 mode 2 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.710+0000 7efd455b1b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195437713797, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 42, "file_size": 2587655, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 21503, "largest_seqno": 23332, "table_properties": {"data_size": 2578672, "index_size": 6300, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1669, "raw_key_size": 14713, "raw_average_key_size": 22, "raw_value_size": 2562757, "raw_average_value_size": 3865, "num_data_blocks": 274, "num_entries": 663, "num_filter_entries": 663, "num_deletions": 5, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1722195437, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8048c795-9f83-4f2e-a5ed-755167692c00", "db_session_id": "SMGEQQAIB6DT6Y2QQ2FS", "orig_file_number": 42, "seqno_to_time_mapping": "N/A"}} 2024-07-28T19:37:18.015 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.710+0000 7efd455b1b00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195437713930, "job": 1, "event": "recovery_finished"} 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.710+0000 7efd455b1b00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 44 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.710+0000 7efd455b1b00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000037.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561b1b3dee00 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 4 rocksdb: DB pointer 0x561b1b4d6000 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd3b37c640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd3b37c640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: ** DB Stats ** 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:37:18.016 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: ** Compaction Stats [default] ** 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: L0 1/0 2.47 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 539.4 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: L6 1/0 11.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Sum 2/0 14.15 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 539.4 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 539.4 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: ** Compaction Stats [default] ** 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-28T19:37:18.017 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 539.4 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Flush(GB): cumulative 0.002, interval 0.002 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Cumulative compaction: 0.00 GB write, 143.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Interval compaction: 0.00 GB write, 143.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:37:18.018 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Block cache BinnedLRUCache@0x561b1b3b31f0#7 capacity: 512.00 MB usage: 8.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.9e-05 secs_since: 0 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,6.58 KB,0.00125468%) Misc(1,0.00 KB,0%) 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 0 starting mon.c rank 2 at public addrs [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] at bind addrs [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 1 mon.c@-1(???) e3 preinit fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 0 mon.c@-1(???).mds e10 new map 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 0 mon.c@-1(???).mds e10 print_map 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: e10 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: legacy client fscid: 1 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Filesystem 'foo' (1) 2024-07-28T19:37:18.019 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: fs_name foo 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: epoch 10 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: created 2024-07-28T19:29:15.255828+0000 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: modified 2024-07-28T19:29:25.471713+0000 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: tableserver 0 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: root 0 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: session_timeout 60 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: session_autoclose 300 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: max_file_size 1099511627776 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: max_xattr_size 65536 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: required_client_features {} 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: last_failure 0 2024-07-28T19:37:18.020 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: last_failure_osd_epoch 70 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: max_mds 1 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: in 0 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: up {0=24331} 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: failed 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: damaged 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: stopped 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: data_pools [4] 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: metadata_pool 3 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: inline_data disabled 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: balancer 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: bal_rank_mask -1 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: standby_count_wanted 1 2024-07-28T19:37:18.021 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: [mds.foo.smithi138.nphkis{0:24331} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] compat {c=[1],r=[1],i=[7ff]}] 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: Standby daemons: 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: [mds.foo.smithi100.ppcqpk{-1:14607} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.100:6834/1539587085,v1:172.21.15.100:6835/1539587085] compat {c=[1],r=[1],i=[7ff]}] 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 0 mon.c@-1(???).osd e386 crush map has features 3314933000854323200, adjusting msgr requires 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 0 mon.c@-1(???).osd e386 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 0 mon.c@-1(???).osd e386 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 0 mon.c@-1(???).osd e386 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:37:18.022 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:17 smithi100 bash[82644]: debug 2024-07-28T19:37:17.714+0000 7efd455b1b00 1 mon.c@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-07-28T19:37:19.040 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.SnapDiscard (5698 ms) 2024-07-28T19:37:19.040 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-07-28T19:37:19.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[60615]: ts=2024-07-28T19:37:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:19.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:15.574527+0000 mgr.y (mgr.25358) 71 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 14 MiB/s wr, 214 op/s 2024-07-28T19:37:19.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.575374+0000 mgr.y (mgr.25358) 72 : cluster [DBG] pgmap v136: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 11 MiB/s wr, 166 op/s 2024-07-28T19:37:19.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.802164+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.803126+0000 mon.a (mon.0) 198 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.823561+0000 mon.a (mon.0) 199 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.835587+0000 mon.a (mon.0) 200 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.836271+0000 mon.a (mon.0) 201 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.836328+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837412+0000 mon.a (mon.0) 203 : cluster [DBG] mgrmap e39: y(active, since 100s), standbys: x 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837596+0000 mon.a (mon.0) 204 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837621+0000 mon.a (mon.0) 205 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837633+0000 mon.a (mon.0) 206 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837646+0000 mon.a (mon.0) 207 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837662+0000 mon.a (mon.0) 208 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837674+0000 mon.a (mon.0) 209 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837685+0000 mon.a (mon.0) 210 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837696+0000 mon.a (mon.0) 211 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837707+0000 mon.a (mon.0) 212 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: cluster 2024-07-28T19:37:17.837719+0000 mon.a (mon.0) 213 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: audit 2024-07-28T19:37:17.867140+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:19.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: audit 2024-07-28T19:37:17.875088+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:19.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:18 smithi138 bash[23952]: audit 2024-07-28T19:37:17.876562+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:19.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:15.574527+0000 mgr.y (mgr.25358) 71 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 14 MiB/s wr, 214 op/s 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.575374+0000 mgr.y (mgr.25358) 72 : cluster [DBG] pgmap v136: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 11 MiB/s wr, 166 op/s 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.802164+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.803126+0000 mon.a (mon.0) 198 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.823561+0000 mon.a (mon.0) 199 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.835587+0000 mon.a (mon.0) 200 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.836271+0000 mon.a (mon.0) 201 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.836328+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837412+0000 mon.a (mon.0) 203 : cluster [DBG] mgrmap e39: y(active, since 100s), standbys: x 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837596+0000 mon.a (mon.0) 204 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837621+0000 mon.a (mon.0) 205 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837633+0000 mon.a (mon.0) 206 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837646+0000 mon.a (mon.0) 207 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837662+0000 mon.a (mon.0) 208 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837674+0000 mon.a (mon.0) 209 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837685+0000 mon.a (mon.0) 210 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837696+0000 mon.a (mon.0) 211 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837707+0000 mon.a (mon.0) 212 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: cluster 2024-07-28T19:37:17.837719+0000 mon.a (mon.0) 213 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: audit 2024-07-28T19:37:17.867140+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: audit 2024-07-28T19:37:17.875088+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[80199]: audit 2024-07-28T19:37:17.876562+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:15.574527+0000 mgr.y (mgr.25358) 71 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 14 MiB/s wr, 214 op/s 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.575374+0000 mgr.y (mgr.25358) 72 : cluster [DBG] pgmap v136: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 11 MiB/s wr, 166 op/s 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.802164+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.803126+0000 mon.a (mon.0) 198 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.823561+0000 mon.a (mon.0) 199 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.835587+0000 mon.a (mon.0) 200 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.836271+0000 mon.a (mon.0) 201 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.836328+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837412+0000 mon.a (mon.0) 203 : cluster [DBG] mgrmap e39: y(active, since 100s), standbys: x 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837596+0000 mon.a (mon.0) 204 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837621+0000 mon.a (mon.0) 205 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837633+0000 mon.a (mon.0) 206 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837646+0000 mon.a (mon.0) 207 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837662+0000 mon.a (mon.0) 208 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837674+0000 mon.a (mon.0) 209 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837685+0000 mon.a (mon.0) 210 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837696+0000 mon.a (mon.0) 211 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837707+0000 mon.a (mon.0) 212 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: cluster 2024-07-28T19:37:17.837719+0000 mon.a (mon.0) 213 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: audit 2024-07-28T19:37:17.867140+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: audit 2024-07-28T19:37:17.875088+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:19.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:18 smithi100 bash[82644]: audit 2024-07-28T19:37:17.876562+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:19.894 INFO:teuthology.orchestra.run.smithi100.stdout:wait for majority of mons to upgrade 2024-07-28T19:37:20.094 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneDiscard (1054 ms) 2024-07-28T19:37:20.094 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-07-28T19:37:20.146 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:19 smithi100 bash[80199]: cluster 2024-07-28T19:37:18.849938+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-07-28T19:37:20.146 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:19 smithi100 bash[82644]: cluster 2024-07-28T19:37:18.849938+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-07-28T19:37:20.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:19 smithi138 bash[23952]: cluster 2024-07-28T19:37:18.849938+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-07-28T19:37:20.685 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-07-28T19:37:20.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:20 smithi100 bash[80199]: cluster 2024-07-28T19:37:19.576144+0000 mgr.y (mgr.25358) 73 : cluster [DBG] pgmap v139: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.0 KiB/s wr, 16 op/s 2024-07-28T19:37:20.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:20 smithi100 bash[80199]: cluster 2024-07-28T19:37:19.859154+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-07-28T19:37:20.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:20 smithi100 bash[82644]: cluster 2024-07-28T19:37:19.576144+0000 mgr.y (mgr.25358) 73 : cluster [DBG] pgmap v139: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.0 KiB/s wr, 16 op/s 2024-07-28T19:37:20.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:20 smithi100 bash[82644]: cluster 2024-07-28T19:37:19.859154+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-07-28T19:37:21.165 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneShrink (1071 ms) 2024-07-28T19:37:21.165 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-07-28T19:37:21.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:20 smithi138 bash[23952]: cluster 2024-07-28T19:37:19.576144+0000 mgr.y (mgr.25358) 73 : cluster [DBG] pgmap v139: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.0 KiB/s wr, 16 op/s 2024-07-28T19:37:21.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:20 smithi138 bash[23952]: cluster 2024-07-28T19:37:19.859154+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-07-28T19:37:22.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:21 smithi100 bash[80199]: cluster 2024-07-28T19:37:20.864949+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-07-28T19:37:22.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:21 smithi100 bash[82644]: cluster 2024-07-28T19:37:20.864949+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-07-28T19:37:22.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:21 smithi138 bash[23952]: cluster 2024-07-28T19:37:20.864949+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-07-28T19:37:22.351 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneExpand (1186 ms) 2024-07-28T19:37:22.351 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-07-28T19:37:22.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:22 smithi138 bash[60615]: ts=2024-07-28T19:37:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:22 smithi100 bash[80199]: cluster 2024-07-28T19:37:21.576823+0000 mgr.y (mgr.25358) 74 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 3.0 KiB/s wr, 25 op/s 2024-07-28T19:37:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:22 smithi100 bash[80199]: cluster 2024-07-28T19:37:21.876135+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-07-28T19:37:22.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:22 smithi100 bash[80199]: audit 2024-07-28T19:37:22.627243+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:22.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:22 smithi100 bash[73305]: debug 2024-07-28T19:37:22.722+0000 7f9676ff2640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-07-28T19:37:22.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:37:22] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.43.0" 2024-07-28T19:37:22.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:22 smithi100 bash[82644]: cluster 2024-07-28T19:37:21.576823+0000 mgr.y (mgr.25358) 74 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 3.0 KiB/s wr, 25 op/s 2024-07-28T19:37:22.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:22 smithi100 bash[82644]: cluster 2024-07-28T19:37:21.876135+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-07-28T19:37:22.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:22 smithi100 bash[82644]: audit 2024-07-28T19:37:22.627243+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:23.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:22 smithi138 bash[23952]: cluster 2024-07-28T19:37:21.576823+0000 mgr.y (mgr.25358) 74 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 3.0 KiB/s wr, 25 op/s 2024-07-28T19:37:23.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:22 smithi138 bash[23952]: cluster 2024-07-28T19:37:21.876135+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-07-28T19:37:23.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:22 smithi138 bash[23952]: audit 2024-07-28T19:37:22.627243+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:23.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:23 smithi138 bash[60615]: ts=2024-07-28T19:37:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:24.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:23 smithi100 bash[80199]: cluster 2024-07-28T19:37:22.885876+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-07-28T19:37:24.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:23 smithi100 bash[80199]: audit 2024-07-28T19:37:23.402143+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:24.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:23 smithi100 bash[80199]: audit 2024-07-28T19:37:23.408035+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:24.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:23 smithi100 bash[82644]: cluster 2024-07-28T19:37:22.885876+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-07-28T19:37:24.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:23 smithi100 bash[82644]: audit 2024-07-28T19:37:23.402143+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:24.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:23 smithi100 bash[82644]: audit 2024-07-28T19:37:23.408035+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:24.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:23 smithi138 bash[23952]: cluster 2024-07-28T19:37:22.885876+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-07-28T19:37:24.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:23 smithi138 bash[23952]: audit 2024-07-28T19:37:23.402143+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:24.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:23 smithi138 bash[23952]: audit 2024-07-28T19:37:23.408035+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:24.749 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:37:24.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:24 smithi138 bash[60615]: ts=2024-07-28T19:37:24.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:24.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:24 smithi138 bash[60615]: ts=2024-07-28T19:37:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:25.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:24 smithi100 bash[80199]: cluster 2024-07-28T19:37:23.578124+0000 mgr.y (mgr.25358) 75 : cluster [DBG] pgmap v145: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-07-28T19:37:25.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:24 smithi100 bash[80199]: cluster 2024-07-28T19:37:23.896085+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-07-28T19:37:25.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:24 smithi100 bash[80199]: audit 2024-07-28T19:37:24.162631+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:25.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:24 smithi100 bash[80199]: audit 2024-07-28T19:37:24.168103+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:24 smithi100 bash[82644]: cluster 2024-07-28T19:37:23.578124+0000 mgr.y (mgr.25358) 75 : cluster [DBG] pgmap v145: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-07-28T19:37:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:24 smithi100 bash[82644]: cluster 2024-07-28T19:37:23.896085+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-07-28T19:37:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:24 smithi100 bash[82644]: audit 2024-07-28T19:37:24.162631+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:25.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:24 smithi100 bash[82644]: audit 2024-07-28T19:37:24.168103+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:24 smithi138 bash[23952]: cluster 2024-07-28T19:37:23.578124+0000 mgr.y (mgr.25358) 75 : cluster [DBG] pgmap v145: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-07-28T19:37:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:24 smithi138 bash[23952]: cluster 2024-07-28T19:37:23.896085+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-07-28T19:37:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:24 smithi138 bash[23952]: audit 2024-07-28T19:37:24.162631+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:24 smithi138 bash[23952]: audit 2024-07-28T19:37:24.168103+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:26.161 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:25 smithi100 bash[80199]: cluster 2024-07-28T19:37:24.900108+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-07-28T19:37:26.162 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:25 smithi100 bash[82644]: cluster 2024-07-28T19:37:24.900108+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-07-28T19:37:26.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:25 smithi138 bash[23952]: cluster 2024-07-28T19:37:24.900108+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-07-28T19:37:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:26 smithi100 bash[80199]: cluster 2024-07-28T19:37:25.578832+0000 mgr.y (mgr.25358) 76 : cluster [DBG] pgmap v148: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-07-28T19:37:27.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:26 smithi100 bash[80199]: cluster 2024-07-28T19:37:25.914580+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-07-28T19:37:27.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:26 smithi100 bash[82644]: cluster 2024-07-28T19:37:25.578832+0000 mgr.y (mgr.25358) 76 : cluster [DBG] pgmap v148: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-07-28T19:37:27.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:26 smithi100 bash[82644]: cluster 2024-07-28T19:37:25.914580+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-07-28T19:37:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:26 smithi138 bash[23952]: cluster 2024-07-28T19:37:25.578832+0000 mgr.y (mgr.25358) 76 : cluster [DBG] pgmap v148: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 406 KiB/s rd, 32 KiB/s wr, 550 op/s 2024-07-28T19:37:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:26 smithi138 bash[23952]: cluster 2024-07-28T19:37:25.914580+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-07-28T19:37:28.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:27 smithi100 bash[82644]: cluster 2024-07-28T19:37:26.924482+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-07-28T19:37:28.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:27 smithi100 bash[80199]: cluster 2024-07-28T19:37:26.924482+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-07-28T19:37:28.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:27 smithi138 bash[23952]: cluster 2024-07-28T19:37:26.924482+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-07-28T19:37:28.541 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable)": 2 2024-07-28T19:37:28.924 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:28 smithi138 bash[60615]: ts=2024-07-28T19:37:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:29.173 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneHideParent (6823 ms) 2024-07-28T19:37:29.174 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Clone 2024-07-28T19:37:29.241 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:28 smithi100 bash[80199]: cluster 2024-07-28T19:37:27.579520+0000 mgr.y (mgr.25358) 77 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:29.241 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:28 smithi100 bash[80199]: cluster 2024-07-28T19:37:27.934542+0000 mon.a (mon.0) 231 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-07-28T19:37:29.241 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:28 smithi100 bash[80199]: audit 2024-07-28T19:37:28.535611+0000 mon.a (mon.0) 232 : audit [DBG] from='client.? 172.21.15.100:0/1782819393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:37:29.241 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:28 smithi100 bash[82644]: cluster 2024-07-28T19:37:27.579520+0000 mgr.y (mgr.25358) 77 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:29.242 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:28 smithi100 bash[82644]: cluster 2024-07-28T19:37:27.934542+0000 mon.a (mon.0) 231 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-07-28T19:37:29.242 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:28 smithi100 bash[82644]: audit 2024-07-28T19:37:28.535611+0000 mon.a (mon.0) 232 : audit [DBG] from='client.? 172.21.15.100:0/1782819393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:37:29.243 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch ps' 2024-07-28T19:37:29.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:28 smithi138 bash[23952]: cluster 2024-07-28T19:37:27.579520+0000 mgr.y (mgr.25358) 77 : cluster [DBG] pgmap v151: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:37:29.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:28 smithi138 bash[23952]: cluster 2024-07-28T19:37:27.934542+0000 mon.a (mon.0) 231 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-07-28T19:37:29.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:28 smithi138 bash[23952]: audit 2024-07-28T19:37:28.535611+0000 mon.a (mon.0) 232 : audit [DBG] from='client.? 172.21.15.100:0/1782819393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:37:30.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:29 smithi100 bash[80199]: cluster 2024-07-28T19:37:28.944184+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-07-28T19:37:30.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:29 smithi100 bash[82644]: cluster 2024-07-28T19:37:28.944184+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-07-28T19:37:30.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:29 smithi138 bash[23952]: cluster 2024-07-28T19:37:28.944184+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-07-28T19:37:31.230 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:30 smithi100 bash[80199]: cluster 2024-07-28T19:37:29.580218+0000 mgr.y (mgr.25358) 78 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-07-28T19:37:31.230 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:30 smithi100 bash[80199]: cluster 2024-07-28T19:37:29.955227+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-07-28T19:37:31.231 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:30 smithi100 bash[82644]: cluster 2024-07-28T19:37:29.580218+0000 mgr.y (mgr.25358) 78 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-07-28T19:37:31.231 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:30 smithi100 bash[82644]: cluster 2024-07-28T19:37:29.955227+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-07-28T19:37:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:30 smithi138 bash[23952]: cluster 2024-07-28T19:37:29.580218+0000 mgr.y (mgr.25358) 78 : cluster [DBG] pgmap v154: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-07-28T19:37:31.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:30 smithi138 bash[23952]: cluster 2024-07-28T19:37:29.955227+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-07-28T19:37:31.442 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Clone (2269 ms) 2024-07-28T19:37:31.442 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-07-28T19:37:32.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:31 smithi100 bash[80199]: cluster 2024-07-28T19:37:30.958304+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-07-28T19:37:32.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:31 smithi100 bash[82644]: cluster 2024-07-28T19:37:30.958304+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-07-28T19:37:32.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:31 smithi138 bash[23952]: cluster 2024-07-28T19:37:30.958304+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-07-28T19:37:32.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:32 smithi138 bash[60615]: ts=2024-07-28T19:37:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:33.066 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:37:33.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:37:32] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.43.0" 2024-07-28T19:37:33.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:32 smithi100 bash[80199]: cluster 2024-07-28T19:37:31.580691+0000 mgr.y (mgr.25358) 79 : cluster [DBG] pgmap v157: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-07-28T19:37:33.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:32 smithi100 bash[80199]: cluster 2024-07-28T19:37:31.977380+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-07-28T19:37:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:32 smithi100 bash[82644]: cluster 2024-07-28T19:37:31.580691+0000 mgr.y (mgr.25358) 79 : cluster [DBG] pgmap v157: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-07-28T19:37:33.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:32 smithi100 bash[82644]: cluster 2024-07-28T19:37:31.977380+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-07-28T19:37:33.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:32 smithi138 bash[23952]: cluster 2024-07-28T19:37:31.580691+0000 mgr.y (mgr.25358) 79 : cluster [DBG] pgmap v157: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 9.5 KiB/s wr, 122 op/s 2024-07-28T19:37:33.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:32 smithi138 bash[23952]: cluster 2024-07-28T19:37:31.977380+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-07-28T19:37:33.694 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneFlatten (2251 ms) 2024-07-28T19:37:33.694 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Stress 2024-07-28T19:37:33.752 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 75018817~4902362 2024-07-28T19:37:33.785 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 72305253~1693883 2024-07-28T19:37:33.788 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 32089748~3660622 2024-07-28T19:37:33.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:33 smithi138 bash[60615]: ts=2024-07-28T19:37:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:33.808 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 19911656~1833194 2024-07-28T19:37:33.809 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 4675829~2966944 2024-07-28T19:37:33.830 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 50128920~1029684 2024-07-28T19:37:33.831 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 1906643~3135921 2024-07-28T19:37:33.850 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 77977279~5908801 2024-07-28T19:37:33.856 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap0 2024-07-28T19:37:34.011 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 95825150 2024-07-28T19:37:34.026 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 13761822~5448342 2024-07-28T19:37:34.033 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 65582074~869996 2024-07-28T19:37:34.040 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 6302708~6103743 2024-07-28T19:37:34.041 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 22238031~1295465 2024-07-28T19:37:34.042 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 87399724~3471315 2024-07-28T19:37:34.042 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 58761752~2813954 2024-07-28T19:37:34.055 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 75569993~7478236 2024-07-28T19:37:34.055 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 11699531~5635756 2024-07-28T19:37:34.093 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:37:34.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:33 smithi100 bash[82644]: cluster 2024-07-28T19:37:32.979360+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-07-28T19:37:34.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:33 smithi100 bash[80199]: cluster 2024-07-28T19:37:32.979360+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-07-28T19:37:34.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:33 smithi138 bash[23952]: cluster 2024-07-28T19:37:32.979360+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-07-28T19:37:34.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:34 smithi138 bash[60615]: ts=2024-07-28T19:37:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:34.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:34 smithi138 bash[60615]: ts=2024-07-28T19:37:34.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:35.030 INFO:tasks.workunit.client.0.smithi100.stdout:clone image236 -> image237 2024-07-28T19:37:35.106 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 6867584~6119211 2024-07-28T19:37:35.147 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 58863911~7138017 2024-07-28T19:37:35.154 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 50478603~1771438 2024-07-28T19:37:35.167 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 42383249~5542354 2024-07-28T19:37:35.176 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 18982145~4455482 2024-07-28T19:37:35.207 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 91793073~4032077 2024-07-28T19:37:35.240 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 68513266~7392989 2024-07-28T19:37:35.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:34 smithi100 bash[80199]: cluster 2024-07-28T19:37:33.581988+0000 mgr.y (mgr.25358) 80 : cluster [DBG] pgmap v160: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 36 MiB/s wr, 553 op/s 2024-07-28T19:37:35.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:34 smithi100 bash[80199]: cluster 2024-07-28T19:37:33.998944+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-07-28T19:37:35.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:34 smithi100 bash[82644]: cluster 2024-07-28T19:37:33.581988+0000 mgr.y (mgr.25358) 80 : cluster [DBG] pgmap v160: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 36 MiB/s wr, 553 op/s 2024-07-28T19:37:35.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:34 smithi100 bash[82644]: cluster 2024-07-28T19:37:33.998944+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-07-28T19:37:35.284 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 46655563~2262489 2024-07-28T19:37:35.286 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:37:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:34 smithi138 bash[23952]: cluster 2024-07-28T19:37:33.581988+0000 mgr.y (mgr.25358) 80 : cluster [DBG] pgmap v160: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 36 MiB/s wr, 553 op/s 2024-07-28T19:37:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:34 smithi138 bash[23952]: cluster 2024-07-28T19:37:33.998944+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-07-28T19:37:36.053 INFO:tasks.workunit.client.0.smithi100.stdout:clone image237 -> image238 2024-07-28T19:37:36.195 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 136844352 2024-07-28T19:37:36.208 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 122566362~4760863 2024-07-28T19:37:36.238 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 39790753~1259793 2024-07-28T19:37:36.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:36 smithi100 bash[82644]: cluster 2024-07-28T19:37:35.007485+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-07-28T19:37:36.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:36 smithi100 bash[80199]: cluster 2024-07-28T19:37:35.007485+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-07-28T19:37:36.248 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 128904548~7939804 2024-07-28T19:37:36.284 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 20698655~6426365 2024-07-28T19:37:36.304 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:36 smithi138 bash[23952]: cluster 2024-07-28T19:37:35.007485+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-07-28T19:37:36.309 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 79347455~2448814 2024-07-28T19:37:36.351 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 52207022~7882417 2024-07-28T19:37:36.370 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 77259244~54378 2024-07-28T19:37:36.373 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 83128587~5369295 2024-07-28T19:37:36.390 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap3 2024-07-28T19:37:37.232 INFO:teuthology.orchestra.run.smithi100.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-28T19:37:37.233 INFO:teuthology.orchestra.run.smithi100.stdout:alertmanager.a smithi100 *:9093,9094 running (3m) 13s ago 10m 13.8M - 0.25.0 c8568f914cd2 61edf7ffcf9d 2024-07-28T19:37:37.233 INFO:teuthology.orchestra.run.smithi100.stdout:grafana.a smithi138 *:3000 running (3m) 35s ago 10m 75.1M - 9.4.7 954c08fa6188 fe0244c1bc8a 2024-07-28T19:37:37.233 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi100.ppcqpk smithi100 running (8m) 13s ago 8m 14.0M - 17.2.7-1155-g3120d060 5ec745195f7e 24cab9a6a84c 2024-07-28T19:37:37.233 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi138.nphkis smithi138 running (8m) 35s ago 8m 16.8M - 17.2.7-1155-g3120d060 5ec745195f7e cc36fa0642f5 2024-07-28T19:37:37.233 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.x smithi138 *:8443,9283,8765 running (2m) 35s ago 18m 423M - 18.2.2-1765-g1c7dd747 dd66d6cc220c 6027bd7b0eab 2024-07-28T19:37:37.233 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.y smithi100 *:8443,9283,8765 running (3m) 13s ago 21m 475M - 18.2.2-1765-g1c7dd747 dd66d6cc220c d4cdb32af700 2024-07-28T19:37:37.234 INFO:teuthology.orchestra.run.smithi100.stdout:mon.a smithi100 running (2m) 13s ago 21m 45.0M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 82b65d214966 2024-07-28T19:37:37.234 INFO:teuthology.orchestra.run.smithi100.stdout:mon.b smithi138 running (19m) 35s ago 19m 51.8M 2048M 17.2.7-1155-g3120d060 5ec745195f7e a76f97e1f96b 2024-07-28T19:37:37.234 INFO:teuthology.orchestra.run.smithi100.stdout:mon.c smithi100 running (21s) 13s ago 19m 19.2M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 7a8870df1475 2024-07-28T19:37:37.234 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.a smithi100 *:9100 running (3m) 13s ago 10m 8160k - 1.5.0 0da6a335fe13 558573030b79 2024-07-28T19:37:37.234 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.b smithi138 *:9100 running (3m) 35s ago 10m 7927k - 1.5.0 0da6a335fe13 3867a7654990 2024-07-28T19:37:37.235 INFO:teuthology.orchestra.run.smithi100.stdout:osd.0 smithi100 running (18m) 13s ago 18m 116M 2524M 17.2.7-1155-g3120d060 5ec745195f7e bc95b51f52b0 2024-07-28T19:37:37.235 INFO:teuthology.orchestra.run.smithi100.stdout:osd.1 smithi100 running (17m) 13s ago 17m 110M 2524M 17.2.7-1155-g3120d060 5ec745195f7e ceb5589accd5 2024-07-28T19:37:37.235 INFO:teuthology.orchestra.run.smithi100.stdout:osd.2 smithi100 running (16m) 13s ago 16m 130M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 1b4df5053189 2024-07-28T19:37:37.235 INFO:teuthology.orchestra.run.smithi100.stdout:osd.3 smithi100 running (15m) 13s ago 15m 112M 2524M 17.2.7-1155-g3120d060 5ec745195f7e cf8d59f38b6d 2024-07-28T19:37:37.235 INFO:teuthology.orchestra.run.smithi100.stdout:osd.4 smithi138 running (14m) 35s ago 14m 100M 2524M 17.2.7-1155-g3120d060 5ec745195f7e feff0c636bab 2024-07-28T19:37:37.236 INFO:teuthology.orchestra.run.smithi100.stdout:osd.5 smithi138 running (13m) 35s ago 13m 98.7M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 33bea4a7c9ec 2024-07-28T19:37:37.236 INFO:teuthology.orchestra.run.smithi100.stdout:osd.6 smithi138 running (13m) 35s ago 13m 118M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 11e0eaf60c0b 2024-07-28T19:37:37.236 INFO:teuthology.orchestra.run.smithi100.stdout:osd.7 smithi138 running (12m) 35s ago 12m 141M 2524M 17.2.7-1155-g3120d060 5ec745195f7e fd79c6c44f0f 2024-07-28T19:37:37.236 INFO:teuthology.orchestra.run.smithi100.stdout:prometheus.a smithi138 *:9095 running (2m) 35s ago 11m 53.8M - 2.43.0 a07b618ecd1d bc318a7bc21e 2024-07-28T19:37:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: cluster 2024-07-28T19:37:35.582666+0000 mgr.y (mgr.25358) 81 : cluster [DBG] pgmap v163: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 36 MiB/s wr, 553 op/s 2024-07-28T19:37:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: cluster 2024-07-28T19:37:36.016981+0000 mon.a (mon.0) 240 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-07-28T19:37:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.217515+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.226901+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.228525+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.229859+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:37.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.240197+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.369652+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.372996+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.374394+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[82644]: audit 2024-07-28T19:37:36.375125+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: cluster 2024-07-28T19:37:35.582666+0000 mgr.y (mgr.25358) 81 : cluster [DBG] pgmap v163: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 36 MiB/s wr, 553 op/s 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: cluster 2024-07-28T19:37:36.016981+0000 mon.a (mon.0) 240 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.217515+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.226901+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.228525+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:37.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.229859+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.240197+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.369652+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.372996+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:37:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.374394+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:37:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:37 smithi100 bash[80199]: audit 2024-07-28T19:37:36.375125+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-07-28T19:37:37.265 INFO:tasks.workunit.client.0.smithi100.stdout:clone image238 -> image239 2024-07-28T19:37:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: cluster 2024-07-28T19:37:35.582666+0000 mgr.y (mgr.25358) 81 : cluster [DBG] pgmap v163: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 36 MiB/s wr, 553 op/s 2024-07-28T19:37:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: cluster 2024-07-28T19:37:36.016981+0000 mon.a (mon.0) 240 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-07-28T19:37:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.217515+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.226901+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.228525+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.229859+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:37:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.240197+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.369652+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:37.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.372996+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:37:37.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.374394+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-07-28T19:37:37.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:37 smithi138 bash[23952]: audit 2024-07-28T19:37:36.375125+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-07-28T19:37:37.888 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Stress (4195 ms) 2024-07-28T19:37:37.888 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-07-28T19:37:37.955 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch upgrade pause' 2024-07-28T19:37:38.241 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:38 smithi100 bash[82644]: cephadm 2024-07-28T19:37:36.375654+0000 mgr.y (mgr.25358) 82 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-07-28T19:37:38.241 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:38 smithi100 bash[82644]: audit 2024-07-28T19:37:37.213910+0000 mgr.y (mgr.25358) 83 : audit [DBG] from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:38.241 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:38 smithi100 bash[82644]: cluster 2024-07-28T19:37:37.243181+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-07-28T19:37:38.241 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:38 smithi100 bash[82644]: audit 2024-07-28T19:37:37.628025+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:38.263 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (375 ms) 2024-07-28T19:37:38.263 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-07-28T19:37:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:38 smithi100 bash[80199]: cephadm 2024-07-28T19:37:36.375654+0000 mgr.y (mgr.25358) 82 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-07-28T19:37:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:38 smithi100 bash[80199]: audit 2024-07-28T19:37:37.213910+0000 mgr.y (mgr.25358) 83 : audit [DBG] from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:38 smithi100 bash[80199]: cluster 2024-07-28T19:37:37.243181+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-07-28T19:37:38.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:38 smithi100 bash[80199]: audit 2024-07-28T19:37:37.628025+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:38.551 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:38 smithi138 bash[23952]: cephadm 2024-07-28T19:37:36.375654+0000 mgr.y (mgr.25358) 82 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-07-28T19:37:38.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:38 smithi138 bash[23952]: audit 2024-07-28T19:37:37.213910+0000 mgr.y (mgr.25358) 83 : audit [DBG] from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:38.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:38 smithi138 bash[23952]: cluster 2024-07-28T19:37:37.243181+0000 mon.a (mon.0) 250 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-07-28T19:37:38.552 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:38 smithi138 bash[23952]: audit 2024-07-28T19:37:37.628025+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:38.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:38 smithi138 bash[60615]: ts=2024-07-28T19:37:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:39.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:39 smithi138 bash[23952]: cluster 2024-07-28T19:37:37.583319+0000 mgr.y (mgr.25358) 84 : cluster [DBG] pgmap v166: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:37:39.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:39 smithi138 bash[23952]: audit 2024-07-28T19:37:39.035698+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:39.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:39 smithi138 bash[23952]: audit 2024-07-28T19:37:39.037773+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:37:39.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:39 smithi138 bash[23952]: audit 2024-07-28T19:37:39.039520+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:37:39.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:39 smithi138 bash[23952]: audit 2024-07-28T19:37:39.041285+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[80199]: cluster 2024-07-28T19:37:37.583319+0000 mgr.y (mgr.25358) 84 : cluster [DBG] pgmap v166: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:37:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[80199]: audit 2024-07-28T19:37:39.035698+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[80199]: audit 2024-07-28T19:37:39.037773+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:37:39.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[80199]: audit 2024-07-28T19:37:39.039520+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:37:39.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[80199]: audit 2024-07-28T19:37:39.041285+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:39.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[82644]: cluster 2024-07-28T19:37:37.583319+0000 mgr.y (mgr.25358) 84 : cluster [DBG] pgmap v166: 105 pgs: 105 active+clean; 209 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:37:39.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[82644]: audit 2024-07-28T19:37:39.035698+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:39.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[82644]: audit 2024-07-28T19:37:39.037773+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:37:39.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[82644]: audit 2024-07-28T19:37:39.039520+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:37:39.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:39 smithi100 bash[82644]: audit 2024-07-28T19:37:39.041285+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:37:40.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:40 smithi138 bash[23952]: cephadm 2024-07-28T19:37:39.026264+0000 mgr.y (mgr.25358) 85 : cephadm [INF] Upgrade: Updating mon.b 2024-07-28T19:37:40.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:40 smithi138 bash[23952]: cephadm 2024-07-28T19:37:39.043388+0000 mgr.y (mgr.25358) 86 : cephadm [INF] Deploying daemon mon.b on smithi138 2024-07-28T19:37:40.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:40 smithi138 bash[23952]: cluster 2024-07-28T19:37:39.255681+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-07-28T19:37:40.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:40 smithi100 bash[80199]: cephadm 2024-07-28T19:37:39.026264+0000 mgr.y (mgr.25358) 85 : cephadm [INF] Upgrade: Updating mon.b 2024-07-28T19:37:40.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:40 smithi100 bash[80199]: cephadm 2024-07-28T19:37:39.043388+0000 mgr.y (mgr.25358) 86 : cephadm [INF] Deploying daemon mon.b on smithi138 2024-07-28T19:37:40.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:40 smithi100 bash[80199]: cluster 2024-07-28T19:37:39.255681+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-07-28T19:37:40.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:40 smithi100 bash[82644]: cephadm 2024-07-28T19:37:39.026264+0000 mgr.y (mgr.25358) 85 : cephadm [INF] Upgrade: Updating mon.b 2024-07-28T19:37:40.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:40 smithi100 bash[82644]: cephadm 2024-07-28T19:37:39.043388+0000 mgr.y (mgr.25358) 86 : cephadm [INF] Deploying daemon mon.b on smithi138 2024-07-28T19:37:40.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:40 smithi100 bash[82644]: cluster 2024-07-28T19:37:39.255681+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-07-28T19:37:41.540 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:41 smithi138 bash[23952]: cluster 2024-07-28T19:37:39.583940+0000 mgr.y (mgr.25358) 87 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s rd, 29 MiB/s wr, 556 op/s 2024-07-28T19:37:41.540 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:41 smithi138 bash[23952]: cluster 2024-07-28T19:37:40.276791+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-07-28T19:37:41.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:41 smithi100 bash[80199]: cluster 2024-07-28T19:37:39.583940+0000 mgr.y (mgr.25358) 87 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s rd, 29 MiB/s wr, 556 op/s 2024-07-28T19:37:41.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:41 smithi100 bash[80199]: cluster 2024-07-28T19:37:40.276791+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-07-28T19:37:41.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:41 smithi100 bash[82644]: cluster 2024-07-28T19:37:39.583940+0000 mgr.y (mgr.25358) 87 : cluster [DBG] pgmap v168: 105 pgs: 105 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 47 MiB/s rd, 29 MiB/s wr, 556 op/s 2024-07-28T19:37:41.744 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:41 smithi100 bash[82644]: cluster 2024-07-28T19:37:40.276791+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-07-28T19:37:41.805 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:41.805 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:41.805 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:41.805 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:41.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:41.806 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:41.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:41.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: Stopping Ceph mon.b for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:37:41.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:41 smithi138 bash[23952]: debug 2024-07-28T19:37:41.781+0000 7f2c231aa640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-28T19:37:41.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:41 smithi138 bash[23952]: debug 2024-07-28T19:37:41.781+0000 7f2c231aa640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-07-28T19:37:41.807 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:41.807 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:37:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:42 smithi138 bash[67779]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-b 2024-07-28T19:37:42.718 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:42 smithi138 bash[60615]: ts=2024-07-28T19:37:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:42.757 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:37:42.982 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.982 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.982 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.983 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.983 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:42 smithi138 bash[67838]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-b 2024-07-28T19:37:42.983 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.b.service: Deactivated successfully. 2024-07-28T19:37:42.983 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: Stopped Ceph mon.b for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:37:42.983 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.983 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: Started Ceph mon.b for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:37:42.983 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.984 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.984 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:42.984 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:37:42 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:37:43.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:37:42] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2024-07-28T19:37:43.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:43 smithi138 bash[60615]: ts=2024-07-28T19:37:43.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.061+0000 7f9dff70eb00 0 set uid:gid to 167:167 (ceph:ceph) 2024-07-28T19:37:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.061+0000 7f9dff70eb00 0 ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable), process ceph-mon, pid 7 2024-07-28T19:37:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.061+0000 7f9dff70eb00 0 pidfile_write: ignore empty --pid-file 2024-07-28T19:37:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 0 load: jerasure load: lrc 2024-07-28T19:37:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: RocksDB version: 7.9.2 2024-07-28T19:37:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Git sha 0 2024-07-28T19:37:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Compile date 2024-07-22 20:55:56 2024-07-28T19:37:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: DB SUMMARY 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: DB Session ID: DOJZGDR2ILIQI6FIQ78B 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: CURRENT file: CURRENT 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: IDENTITY file: IDENTITY 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 1846 Bytes 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000039.sst 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000037.log size: 5521895 ; 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.error_if_exists: 0 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.create_if_missing: 0 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.paranoid_checks: 1 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.env: 0x56168841ebc0 2024-07-28T19:37:44.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.fs: PosixFileSystem 2024-07-28T19:37:44.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.info_log: 0x56168a0f4be0 2024-07-28T19:37:44.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_file_opening_threads: 16 2024-07-28T19:37:44.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.statistics: (nil) 2024-07-28T19:37:44.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.use_fsync: 0 2024-07-28T19:37:44.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_log_file_size: 0 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.keep_log_file_num: 1000 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.recycle_log_file_num: 0 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.allow_fallocate: 1 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.allow_mmap_reads: 0 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.allow_mmap_writes: 0 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.use_direct_reads: 0 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.create_missing_column_families: 0 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.db_log_dir: 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.wal_dir: 2024-07-28T19:37:44.309 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.advise_random_on_open: 1 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.db_write_buffer_size: 0 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.write_buffer_manager: 0x56168a102b40 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-07-28T19:37:44.310 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.rate_limiter: (nil) 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.wal_recovery_mode: 2 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.enable_thread_tracking: 0 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.enable_pipelined_write: 0 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.unordered_write: 0 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.row_cache: None 2024-07-28T19:37:44.311 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.wal_filter: None 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.allow_ingest_behind: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.two_write_queues: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.manual_wal_flush: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.wal_compression: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.atomic_flush: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.log_readahead_size: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.best_efforts_recovery: 0 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-07-28T19:37:44.312 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.allow_data_in_errors: 0 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.db_host_id: __hostname__ 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_background_jobs: 2 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_background_compactions: -1 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_subcompactions: 1 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_total_wal_size: 0 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_open_files: -1 2024-07-28T19:37:44.313 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bytes_per_sync: 0 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_readahead_size: 0 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_background_flushes: -1 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Compression algorithms supported: 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: kZSTD supported: 0 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: kXpressCompression supported: 0 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: kBZip2Compression supported: 0 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: kLZ4Compression supported: 1 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: kZlibCompression supported: 1 2024-07-28T19:37:44.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: kLZ4HCCompression supported: 1 2024-07-28T19:37:44.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: kSnappyCompression supported: 1 2024-07-28T19:37:44.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-07-28T19:37:44.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-07-28T19:37:44.315 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-07-28T19:37:44.316 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-07-28T19:37:44.316 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-07-28T19:37:44.316 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.merge_operator: 2024-07-28T19:37:44.316 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_filter: None 2024-07-28T19:37:44.316 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_filter_factory: None 2024-07-28T19:37:44.316 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.sst_partitioner_factory: None 2024-07-28T19:37:44.316 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-07-28T19:37:44.316 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56168a0f4820) 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: cache_index_and_filter_blocks: 1 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: cache_index_and_filter_blocks_with_high_priority: 0 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: pin_top_level_index_and_filter: 1 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: index_type: 0 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: data_block_index_type: 0 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: index_shortening: 1 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: data_block_hash_table_util_ratio: 0.750000 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: checksum: 4 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: no_block_cache: 0 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: block_cache: 0x56168a0e9350 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: block_cache_name: BinnedLRUCache 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: block_cache_options: 2024-07-28T19:37:44.317 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: capacity : 536870912 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: num_shard_bits : 4 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: strict_capacity_limit : 0 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: high_pri_pool_ratio: 0.000 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: block_cache_compressed: (nil) 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: persistent_cache: (nil) 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: block_size: 4096 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: block_size_deviation: 10 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: block_restart_interval: 16 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: index_block_restart_interval: 1 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: metadata_block_size: 4096 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: partition_filters: 0 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: use_delta_encoding: 1 2024-07-28T19:37:44.318 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: filter_policy: bloomfilter 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: whole_key_filtering: 1 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: verify_compression: 0 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: read_amp_bytes_per_bit: 0 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: format_version: 5 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: enable_index_compression: 1 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: block_align: 0 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: max_auto_readahead_size: 262144 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: prepopulate_block_cache: 0 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: initial_auto_readahead_size: 8192 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: num_file_reads_for_auto_readahead: 2 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.write_buffer_size: 33554432 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_write_buffer_number: 2 2024-07-28T19:37:44.319 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression: NoCompression 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression: Disabled 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.prefix_extractor: nullptr 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.num_levels: 7 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-07-28T19:37:44.320 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.level: 32767 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.strategy: 0 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-07-28T19:37:44.321 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.enabled: false 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.target_file_size_base: 67108864 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-07-28T19:37:44.322 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.arena_block_size: 1048576 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-07-28T19:37:44.323 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.disable_auto_compactions: 0 2024-07-28T19:37:44.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-07-28T19:37:44.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-07-28T19:37:44.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-07-28T19:37:44.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-07-28T19:37:44.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-07-28T19:37:44.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-07-28T19:37:44.324 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-07-28T19:37:44.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-07-28T19:37:44.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-07-28T19:37:44.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-07-28T19:37:44.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.table_properties_collectors: 2024-07-28T19:37:44.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.inplace_update_support: 0 2024-07-28T19:37:44.325 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-07-28T19:37:44.326 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-07-28T19:37:44.326 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-07-28T19:37:44.326 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-07-28T19:37:44.326 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.bloom_locality: 0 2024-07-28T19:37:44.326 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.max_successive_merges: 0 2024-07-28T19:37:44.327 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-07-28T19:37:44.327 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.paranoid_file_checks: 0 2024-07-28T19:37:44.327 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.force_consistency_checks: 1 2024-07-28T19:37:44.327 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.report_bg_io_stats: 0 2024-07-28T19:37:44.327 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.ttl: 2592000 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.enable_blob_files: false 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.min_blob_size: 0 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.blob_file_size: 268435456 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-07-28T19:37:44.328 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.blob_file_starting_level: 0 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dfc4e7640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 39.sst 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 41, last_sequence is 21587, log_number is 37,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 37 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b16564dc-38b9-4c58-b6f9-d6ea2c427c1b 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195464071722, "job": 1, "event": "recovery_started", "wal_files": [37]} 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.065+0000 7f9dff70eb00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #37 mode 2 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195464092316, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 42, "file_size": 3419574, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 21592, "largest_seqno": 24070, "table_properties": {"data_size": 3407821, "index_size": 8429, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2309, "raw_key_size": 19966, "raw_average_key_size": 22, "raw_value_size": 3386530, "raw_average_value_size": 3766, "num_data_blocks": 366, "num_entries": 899, "num_filter_entries": 899, "num_deletions": 5, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1722195464, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b16564dc-38b9-4c58-b6f9-d6ea2c427c1b", "db_session_id": "DOJZGDR2ILIQI6FIQ78B", "orig_file_number": 42, "seqno_to_time_mapping": "N/A"}} 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1722195464092459, "job": 1, "event": "recovery_finished"} 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 44 2024-07-28T19:37:44.329 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000037.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56168a114e00 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 4 rocksdb: DB pointer 0x56168a20c000 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9df54d9640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9df54d9640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: ** DB Stats ** 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-07-28T19:37:44.330 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: ** Compaction Stats [default] ** 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: L0 1/0 3.26 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: L6 1/0 11.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Sum 2/0 14.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: ** Compaction Stats [default] ** 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 532.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-07-28T19:37:44.331 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Uptime(secs): 0.0 total, 0.0 interval 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Flush(GB): cumulative 0.003, interval 0.003 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: AddFile(GB): cumulative 0.000, interval 0.000 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: AddFile(Total Files): cumulative 0, interval 0 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: AddFile(L0 Files): cumulative 0, interval 0 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: AddFile(Keys): cumulative 0, interval 0 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Cumulative compaction: 0.00 GB write, 136.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Interval compaction: 0.00 GB write, 136.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Block cache BinnedLRUCache@0x56168a0e9350#7 capacity: 512.00 MB usage: 11.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Block cache entry stats(count,size,portion): FilterBlock(1,2.53 KB,0.000482798%) IndexBlock(1,9.08 KB,0.00173151%) Misc(1,0.00 KB,0%) 2024-07-28T19:37:44.332 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: ** File Read Latency Histogram By Level [default] ** 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 0 starting mon.b rank 1 at public addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] at bind addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.089+0000 7f9dff70eb00 1 mon.b@-1(???) e3 preinit fsid 9170a172-4d15-11ef-bcc8-c7b262605968 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.093+0000 7f9dff70eb00 0 mon.b@-1(???).mds e10 new map 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.093+0000 7f9dff70eb00 0 mon.b@-1(???).mds e10 print_map 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: e10 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: enable_multiple, ever_enabled_multiple: 1,1 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: legacy client fscid: 1 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Filesystem 'foo' (1) 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: fs_name foo 2024-07-28T19:37:44.333 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: epoch 10 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: created 2024-07-28T19:29:15.255828+0000 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: modified 2024-07-28T19:29:25.471713+0000 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: tableserver 0 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: root 0 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: session_timeout 60 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: session_autoclose 300 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: max_file_size 1099511627776 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: max_xattr_size 65536 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: required_client_features {} 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: last_failure 0 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: last_failure_osd_epoch 70 2024-07-28T19:37:44.334 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: max_mds 1 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: in 0 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: up {0=24331} 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: failed 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: damaged 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: stopped 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: data_pools [4] 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: metadata_pool 3 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: inline_data disabled 2024-07-28T19:37:44.335 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: balancer 2024-07-28T19:37:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: bal_rank_mask -1 2024-07-28T19:37:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: standby_count_wanted 1 2024-07-28T19:37:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: [mds.foo.smithi138.nphkis{0:24331} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.138:6832/3322744845,v1:172.21.15.138:6833/3322744845] compat {c=[1],r=[1],i=[7ff]}] 2024-07-28T19:37:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: 2024-07-28T19:37:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: 2024-07-28T19:37:44.336 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: Standby daemons: 2024-07-28T19:37:44.337 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: 2024-07-28T19:37:44.337 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: [mds.foo.smithi100.ppcqpk{-1:14607} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.100:6834/1539587085,v1:172.21.15.100:6835/1539587085] compat {c=[1],r=[1],i=[7ff]}] 2024-07-28T19:37:44.337 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.093+0000 7f9dff70eb00 0 mon.b@-1(???).osd e409 crush map has features 3314933000854323200, adjusting msgr requires 2024-07-28T19:37:44.337 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.093+0000 7f9dff70eb00 0 mon.b@-1(???).osd e409 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:37:44.337 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.093+0000 7f9dff70eb00 0 mon.b@-1(???).osd e409 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:37:44.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.093+0000 7f9dff70eb00 0 mon.b@-1(???).osd e409 crush map has features 432629239337189376, adjusting msgr requires 2024-07-28T19:37:44.338 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[67894]: debug 2024-07-28T19:37:44.093+0000 7f9dff70eb00 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2024-07-28T19:37:44.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[60615]: ts=2024-07-28T19:37:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:44.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:44 smithi138 bash[60615]: ts=2024-07-28T19:37:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:45.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:45.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:45.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:45 smithi100 bash[80199]: debug 2024-07-28T19:37:45.333+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:49.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:48 smithi138 bash[60615]: ts=2024-07-28T19:37:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:49.494 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:49 smithi100 bash[73305]: debug 2024-07-28T19:37:49.105+0000 7f9676ff2640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : mon.c (rank 2) addr [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] is down (out of quorum) 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: debug 2024-07-28T19:37:50.417+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:45.363343+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: audit 2024-07-28T19:37:45.364580+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:45.365184+0000 mon.a (mon.0) 276 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: audit 2024-07-28T19:37:45.387104+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: audit 2024-07-28T19:37:45.387691+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:45.586660+0000 mgr.y (mgr.25358) 90 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 1.3 MiB/s wr, 32 op/s 2024-07-28T19:37:50.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:47.587338+0000 mgr.y (mgr.25358) 91 : cluster [DBG] pgmap v174: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:49.588676+0000 mgr.y (mgr.25358) 92 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 883 KiB/s wr, 21 op/s 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.386612+0000 mon.a (mon.0) 279 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.395328+0000 mon.a (mon.0) 280 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.414039+0000 mon.a (mon.0) 281 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.414118+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.414656+0000 mon.a (mon.0) 283 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.415019+0000 mon.a (mon.0) 284 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424586+0000 mon.a (mon.0) 285 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424626+0000 mon.a (mon.0) 286 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424649+0000 mon.a (mon.0) 287 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] is down (out of quorum) 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424716+0000 mon.a (mon.0) 288 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424754+0000 mon.a (mon.0) 289 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424795+0000 mon.a (mon.0) 290 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424839+0000 mon.a (mon.0) 291 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424887+0000 mon.a (mon.0) 292 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424916+0000 mon.a (mon.0) 293 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424945+0000 mon.a (mon.0) 294 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.424971+0000 mon.a (mon.0) 295 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.425001+0000 mon.a (mon.0) 296 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: audit 2024-07-28T19:37:50.431384+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: audit 2024-07-28T19:37:50.441209+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:50 smithi100 bash[80199]: audit 2024-07-28T19:37:50.443961+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:50.803 INFO:teuthology.orchestra.run.smithi100.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:37:50.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:45.363343+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-28T19:37:50.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: audit 2024-07-28T19:37:45.364580+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:37:50.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:45.365184+0000 mon.a (mon.0) 276 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:50.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: audit 2024-07-28T19:37:45.387104+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:37:50.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: audit 2024-07-28T19:37:45.387691+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:37:50.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:45.586660+0000 mgr.y (mgr.25358) 90 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 1.3 MiB/s wr, 32 op/s 2024-07-28T19:37:50.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:47.587338+0000 mgr.y (mgr.25358) 91 : cluster [DBG] pgmap v174: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:49.588676+0000 mgr.y (mgr.25358) 92 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 883 KiB/s wr, 21 op/s 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.386612+0000 mon.a (mon.0) 279 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.395328+0000 mon.a (mon.0) 280 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.414039+0000 mon.a (mon.0) 281 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.414118+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.414656+0000 mon.a (mon.0) 283 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.415019+0000 mon.a (mon.0) 284 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424586+0000 mon.a (mon.0) 285 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424626+0000 mon.a (mon.0) 286 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424649+0000 mon.a (mon.0) 287 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] is down (out of quorum) 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424716+0000 mon.a (mon.0) 288 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424754+0000 mon.a (mon.0) 289 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424795+0000 mon.a (mon.0) 290 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424839+0000 mon.a (mon.0) 291 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424887+0000 mon.a (mon.0) 292 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424916+0000 mon.a (mon.0) 293 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424945+0000 mon.a (mon.0) 294 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.424971+0000 mon.a (mon.0) 295 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.425001+0000 mon.a (mon.0) 296 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: audit 2024-07-28T19:37:50.431384+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: audit 2024-07-28T19:37:50.441209+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:50 smithi138 bash[67894]: audit 2024-07-28T19:37:50.443961+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:51.599 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'sleep 60' 2024-07-28T19:37:51.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: cluster 2024-07-28T19:37:50.474947+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-07-28T19:37:51.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: audit 2024-07-28T19:37:50.802813+0000 mgr.y (mgr.25358) 93 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:51.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: cephadm 2024-07-28T19:37:50.803215+0000 mgr.y (mgr.25358) 94 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: audit 2024-07-28T19:37:50.803379+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: audit 2024-07-28T19:37:50.808514+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: audit 2024-07-28T19:37:50.812969+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.585+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.585+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.585+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.585+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.585+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.589+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.589+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.589+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.589+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[80199]: debug 2024-07-28T19:37:51.589+0000 7f8fc1d02640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:45.363343+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-07-28T19:37:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:45.364580+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-07-28T19:37:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:45.365184+0000 mon.a (mon.0) 276 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:45.387104+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-07-28T19:37:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:45.387691+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-07-28T19:37:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:45.586660+0000 mgr.y (mgr.25358) 90 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 1.3 MiB/s wr, 32 op/s 2024-07-28T19:37:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:47.587338+0000 mgr.y (mgr.25358) 91 : cluster [DBG] pgmap v174: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-07-28T19:37:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:49.588676+0000 mgr.y (mgr.25358) 92 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 883 KiB/s wr, 21 op/s 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.386612+0000 mon.a (mon.0) 279 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.395328+0000 mon.a (mon.0) 280 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.414039+0000 mon.a (mon.0) 281 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.414118+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.414656+0000 mon.a (mon.0) 283 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.415019+0000 mon.a (mon.0) 284 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424586+0000 mon.a (mon.0) 285 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424626+0000 mon.a (mon.0) 286 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424649+0000 mon.a (mon.0) 287 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0] is down (out of quorum) 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424716+0000 mon.a (mon.0) 288 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424754+0000 mon.a (mon.0) 289 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424795+0000 mon.a (mon.0) 290 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424839+0000 mon.a (mon.0) 291 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424887+0000 mon.a (mon.0) 292 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424916+0000 mon.a (mon.0) 293 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424945+0000 mon.a (mon.0) 294 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.424971+0000 mon.a (mon.0) 295 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.425001+0000 mon.a (mon.0) 296 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:50.431384+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:50.441209+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:50.443961+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cluster 2024-07-28T19:37:50.474947+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:50.802813+0000 mgr.y (mgr.25358) 93 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: cephadm 2024-07-28T19:37:50.803215+0000 mgr.y (mgr.25358) 94 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:50.803379+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:50.808514+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:51 smithi100 bash[82644]: audit 2024-07-28T19:37:50.812969+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:51 smithi138 bash[67894]: cluster 2024-07-28T19:37:50.474947+0000 mon.a (mon.0) 300 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-07-28T19:37:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:51 smithi138 bash[67894]: audit 2024-07-28T19:37:50.802813+0000 mgr.y (mgr.25358) 93 : audit [DBG] from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:37:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:51 smithi138 bash[67894]: cephadm 2024-07-28T19:37:50.803215+0000 mgr.y (mgr.25358) 94 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:37:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:51 smithi138 bash[67894]: audit 2024-07-28T19:37:50.803379+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:51 smithi138 bash[67894]: audit 2024-07-28T19:37:50.808514+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:51.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:51 smithi138 bash[67894]: audit 2024-07-28T19:37:50.812969+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:52.070 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (13806 ms) 2024-07-28T19:37:52.070 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:45.386504+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.514224+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.516035+0000 mon.a (mon.0) 304 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.558438+0000 mon.a (mon.0) 305 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.567244+0000 mon.a (mon.0) 306 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.568680+0000 mon.a (mon.0) 307 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.568753+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.569646+0000 mon.a (mon.0) 309 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.570101+0000 mon.a (mon.0) 310 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-28T19:37:52.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.589470+0000 mgr.y (mgr.25358) 95 : cluster [DBG] pgmap v177: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 822 KiB/s wr, 20 op/s 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.592909+0000 mon.a (mon.0) 311 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.592957+0000 mon.a (mon.0) 312 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.592978+0000 mon.a (mon.0) 313 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.592991+0000 mon.a (mon.0) 314 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.593005+0000 mon.a (mon.0) 315 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.593058+0000 mon.a (mon.0) 316 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.593071+0000 mon.a (mon.0) 317 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.593084+0000 mon.a (mon.0) 318 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.593098+0000 mon.a (mon.0) 319 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[82644]: cluster 2024-07-28T19:37:51.593114+0000 mon.a (mon.0) 320 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:45.386504+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.514224+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.516035+0000 mon.a (mon.0) 304 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.558438+0000 mon.a (mon.0) 305 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.567244+0000 mon.a (mon.0) 306 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.568680+0000 mon.a (mon.0) 307 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.568753+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.569646+0000 mon.a (mon.0) 309 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.570101+0000 mon.a (mon.0) 310 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.589470+0000 mgr.y (mgr.25358) 95 : cluster [DBG] pgmap v177: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 822 KiB/s wr, 20 op/s 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.592909+0000 mon.a (mon.0) 311 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.592957+0000 mon.a (mon.0) 312 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.592978+0000 mon.a (mon.0) 313 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.592991+0000 mon.a (mon.0) 314 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.593005+0000 mon.a (mon.0) 315 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.593058+0000 mon.a (mon.0) 316 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.593071+0000 mon.a (mon.0) 317 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.593084+0000 mon.a (mon.0) 318 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.593098+0000 mon.a (mon.0) 319 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[80199]: cluster 2024-07-28T19:37:51.593114+0000 mon.a (mon.0) 320 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[60615]: ts=2024-07-28T19:37:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:52.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:45.386504+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-07-28T19:37:52.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.514224+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-07-28T19:37:52.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.516035+0000 mon.a (mon.0) 304 : cluster [INF] mon.a calling monitor election 2024-07-28T19:37:52.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.558438+0000 mon.a (mon.0) 305 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-07-28T19:37:52.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.567244+0000 mon.a (mon.0) 306 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0],b=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],c=[v2:172.21.15.100:3301/0,v1:172.21.15.100:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.568680+0000 mon.a (mon.0) 307 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.nphkis=up:active} 1 up:standby 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.568753+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.569646+0000 mon.a (mon.0) 309 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.570101+0000 mon.a (mon.0) 310 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.589470+0000 mgr.y (mgr.25358) 95 : cluster [DBG] pgmap v177: 105 pgs: 105 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 822 KiB/s wr, 20 op/s 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.592909+0000 mon.a (mon.0) 311 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.592957+0000 mon.a (mon.0) 312 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.592978+0000 mon.a (mon.0) 313 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.592991+0000 mon.a (mon.0) 314 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.593005+0000 mon.a (mon.0) 315 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.593058+0000 mon.a (mon.0) 316 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.593071+0000 mon.a (mon.0) 317 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.593084+0000 mon.a (mon.0) 318 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.593098+0000 mon.a (mon.0) 319 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:52.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:52 smithi138 bash[67894]: cluster 2024-07-28T19:37:51.593114+0000 mon.a (mon.0) 320 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:37:53.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:37:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:37:52] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2024-07-28T19:37:53.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:53 smithi138 bash[67894]: cluster 2024-07-28T19:37:52.611727+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-07-28T19:37:53.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:53 smithi138 bash[67894]: audit 2024-07-28T19:37:52.628092+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:53.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:53 smithi138 bash[60615]: ts=2024-07-28T19:37:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:53.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:53 smithi100 bash[80199]: cluster 2024-07-28T19:37:52.611727+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-07-28T19:37:53.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:53 smithi100 bash[80199]: audit 2024-07-28T19:37:52.628092+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:53.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:53 smithi100 bash[82644]: cluster 2024-07-28T19:37:52.611727+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-07-28T19:37:53.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:53 smithi100 bash[82644]: audit 2024-07-28T19:37:52.628092+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:37:54.353 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2283 ms) 2024-07-28T19:37:54.353 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-07-28T19:37:54.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:54 smithi138 bash[67894]: cluster 2024-07-28T19:37:53.590870+0000 mgr.y (mgr.25358) 96 : cluster [DBG] pgmap v179: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 70 op/s 2024-07-28T19:37:54.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:54 smithi138 bash[67894]: cluster 2024-07-28T19:37:53.624909+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-07-28T19:37:54.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:54 smithi138 bash[60615]: ts=2024-07-28T19:37:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:54.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:54 smithi138 bash[60615]: ts=2024-07-28T19:37:54.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:54.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:54 smithi100 bash[80199]: cluster 2024-07-28T19:37:53.590870+0000 mgr.y (mgr.25358) 96 : cluster [DBG] pgmap v179: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 70 op/s 2024-07-28T19:37:54.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:54 smithi100 bash[80199]: cluster 2024-07-28T19:37:53.624909+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-07-28T19:37:54.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:54 smithi100 bash[82644]: cluster 2024-07-28T19:37:53.590870+0000 mgr.y (mgr.25358) 96 : cluster [DBG] pgmap v179: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 70 op/s 2024-07-28T19:37:54.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:54 smithi100 bash[82644]: cluster 2024-07-28T19:37:53.624909+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-07-28T19:37:56.408 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:37:56.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:56 smithi100 bash[80199]: cluster 2024-07-28T19:37:55.138377+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-07-28T19:37:56.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:56 smithi100 bash[82644]: cluster 2024-07-28T19:37:55.138377+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-07-28T19:37:56.554 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:56 smithi138 bash[67894]: cluster 2024-07-28T19:37:55.138377+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-07-28T19:37:56.830 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2477 ms) 2024-07-28T19:37:56.830 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-07-28T19:37:56.887 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 68437417~3457085 2024-07-28T19:37:56.913 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 10850074~3310546 2024-07-28T19:37:56.914 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 30738860~2420365 2024-07-28T19:37:56.931 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 39440196~2335844 2024-07-28T19:37:56.933 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 50455970~5550661 2024-07-28T19:37:56.958 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 2287903~2258019 2024-07-28T19:37:56.959 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 34760581~1384115 2024-07-28T19:37:56.968 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 31751605~1839099 2024-07-28T19:37:56.974 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap0 2024-07-28T19:37:57.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:57 smithi100 bash[80199]: cluster 2024-07-28T19:37:55.591567+0000 mgr.y (mgr.25358) 97 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 7.0 MiB/s wr, 110 op/s 2024-07-28T19:37:57.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:57 smithi100 bash[80199]: cluster 2024-07-28T19:37:56.145816+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-07-28T19:37:57.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:57 smithi100 bash[80199]: audit 2024-07-28T19:37:56.671258+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:57.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:57 smithi100 bash[80199]: audit 2024-07-28T19:37:56.685142+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:57.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:57 smithi100 bash[82644]: cluster 2024-07-28T19:37:55.591567+0000 mgr.y (mgr.25358) 97 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 7.0 MiB/s wr, 110 op/s 2024-07-28T19:37:57.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:57 smithi100 bash[82644]: cluster 2024-07-28T19:37:56.145816+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-07-28T19:37:57.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:57 smithi100 bash[82644]: audit 2024-07-28T19:37:56.671258+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:57.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:57 smithi100 bash[82644]: audit 2024-07-28T19:37:56.685142+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:57.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:57 smithi138 bash[67894]: cluster 2024-07-28T19:37:55.591567+0000 mgr.y (mgr.25358) 97 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 7.0 MiB/s wr, 110 op/s 2024-07-28T19:37:57.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:57 smithi138 bash[67894]: cluster 2024-07-28T19:37:56.145816+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-07-28T19:37:57.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:57 smithi138 bash[67894]: audit 2024-07-28T19:37:56.671258+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:57.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:57 smithi138 bash[67894]: audit 2024-07-28T19:37:56.685142+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:57.715 INFO:tasks.workunit.client.0.smithi100.stdout:clone image258 -> image259 2024-07-28T19:37:57.817 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 18344897~645987 2024-07-28T19:37:57.830 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 74374738~3219404 2024-07-28T19:37:57.844 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 70563832~1622060 2024-07-28T19:37:57.870 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 46025911~4246561 2024-07-28T19:37:57.878 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 24939754~5461970 2024-07-28T19:37:57.905 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 61478702~3697429 2024-07-28T19:37:57.913 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 75740691~8145389 2024-07-28T19:37:57.937 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 78564799~4087000 2024-07-28T19:37:57.940 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:37:58.722 INFO:tasks.workunit.client.0.smithi100.stdout:clone image259 -> image260 2024-07-28T19:37:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:58 smithi100 bash[80199]: audit 2024-07-28T19:37:57.431980+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:58.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:58 smithi100 bash[80199]: audit 2024-07-28T19:37:57.441828+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:58.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:58 smithi100 bash[80199]: cluster 2024-07-28T19:37:57.592225+0000 mgr.y (mgr.25358) 98 : cluster [DBG] pgmap v184: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 MiB/s wr, 113 op/s 2024-07-28T19:37:58.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:58 smithi100 bash[80199]: cluster 2024-07-28T19:37:57.689329+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-07-28T19:37:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:58 smithi100 bash[82644]: audit 2024-07-28T19:37:57.431980+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:58 smithi100 bash[82644]: audit 2024-07-28T19:37:57.441828+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:58 smithi100 bash[82644]: cluster 2024-07-28T19:37:57.592225+0000 mgr.y (mgr.25358) 98 : cluster [DBG] pgmap v184: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 MiB/s wr, 113 op/s 2024-07-28T19:37:58.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:58 smithi100 bash[82644]: cluster 2024-07-28T19:37:57.689329+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-07-28T19:37:58.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:37:58 smithi138 bash[60615]: ts=2024-07-28T19:37:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:37:58.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:58 smithi138 bash[67894]: audit 2024-07-28T19:37:57.431980+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:58.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:58 smithi138 bash[67894]: audit 2024-07-28T19:37:57.441828+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:37:58.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:58 smithi138 bash[67894]: cluster 2024-07-28T19:37:57.592225+0000 mgr.y (mgr.25358) 98 : cluster [DBG] pgmap v184: 105 pgs: 105 active+clean; 396 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 7.2 MiB/s wr, 113 op/s 2024-07-28T19:37:58.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:58 smithi138 bash[67894]: cluster 2024-07-28T19:37:57.689329+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-07-28T19:37:58.861 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 142858091 2024-07-28T19:37:58.873 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 25897530~3563217 2024-07-28T19:37:58.914 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 45127069~5802416 2024-07-28T19:37:58.943 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 6579789~6262541 2024-07-28T19:37:58.971 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 29361699~5071654 2024-07-28T19:37:58.994 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 102925104~1540810 2024-07-28T19:37:59.006 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 53332258~6712596 2024-07-28T19:37:59.013 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 97282151~3914815 2024-07-28T19:37:59.031 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 27202640~97201 2024-07-28T19:37:59.033 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:37:59.736 INFO:tasks.workunit.client.0.smithi100.stdout:clone image260 -> image261 2024-07-28T19:37:59.910 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 129343193 2024-07-28T19:37:59.931 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 123746592~5596601 2024-07-28T19:37:59.966 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 122530620~6812573 2024-07-28T19:37:59.970 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 17453050~3383978 2024-07-28T19:37:59.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:37:59 smithi100 bash[82644]: cluster 2024-07-28T19:37:58.691372+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-07-28T19:37:59.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:37:59 smithi100 bash[80199]: cluster 2024-07-28T19:37:58.691372+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-07-28T19:37:59.997 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 93284833~8059937 2024-07-28T19:38:00.019 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 70502654~1248073 2024-07-28T19:38:00.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:37:59 smithi138 bash[67894]: cluster 2024-07-28T19:37:58.691372+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-07-28T19:38:00.058 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 65094620~5589735 2024-07-28T19:38:00.066 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 92687188~4461919 2024-07-28T19:38:00.089 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 93396000~7302245 2024-07-28T19:38:00.092 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap3 2024-07-28T19:38:00.745 INFO:tasks.workunit.client.0.smithi100.stdout:clone image261 -> image262 2024-07-28T19:38:00.986 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 194256416 2024-07-28T19:38:00.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:00 smithi100 bash[80199]: cluster 2024-07-28T19:37:59.592853+0000 mgr.y (mgr.25358) 99 : cluster [DBG] pgmap v187: 105 pgs: 105 active+clean; 576 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 37 MiB/s wr, 478 op/s 2024-07-28T19:38:00.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:00 smithi100 bash[80199]: cluster 2024-07-28T19:37:59.700380+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-07-28T19:38:00.994 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:00 smithi100 bash[82644]: cluster 2024-07-28T19:37:59.592853+0000 mgr.y (mgr.25358) 99 : cluster [DBG] pgmap v187: 105 pgs: 105 active+clean; 576 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 37 MiB/s wr, 478 op/s 2024-07-28T19:38:00.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:00 smithi100 bash[82644]: cluster 2024-07-28T19:37:59.700380+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-07-28T19:38:01.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:00 smithi138 bash[67894]: cluster 2024-07-28T19:37:59.592853+0000 mgr.y (mgr.25358) 99 : cluster [DBG] pgmap v187: 105 pgs: 105 active+clean; 576 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 37 MiB/s wr, 478 op/s 2024-07-28T19:38:01.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:00 smithi138 bash[67894]: cluster 2024-07-28T19:37:59.700380+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-07-28T19:38:01.662 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (4832 ms) 2024-07-28T19:38:01.662 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-07-28T19:38:01.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:01 smithi100 bash[80199]: cluster 2024-07-28T19:38:00.704345+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-07-28T19:38:01.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:01 smithi100 bash[82644]: cluster 2024-07-28T19:38:00.704345+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-07-28T19:38:02.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:01 smithi138 bash[67894]: cluster 2024-07-28T19:38:00.704345+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-07-28T19:38:02.089 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (427 ms) 2024-07-28T19:38:02.089 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-07-28T19:38:02.714 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:02 smithi138 bash[60615]: ts=2024-07-28T19:38:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:02.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:02 smithi100 bash[80199]: cluster 2024-07-28T19:38:01.593621+0000 mgr.y (mgr.25358) 100 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 576 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 533 op/s 2024-07-28T19:38:02.995 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:38:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:38:02] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2024-07-28T19:38:02.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:02 smithi100 bash[82644]: cluster 2024-07-28T19:38:01.593621+0000 mgr.y (mgr.25358) 100 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 576 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 533 op/s 2024-07-28T19:38:03.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:02 smithi138 bash[67894]: cluster 2024-07-28T19:38:01.593621+0000 mgr.y (mgr.25358) 100 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 576 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 533 op/s 2024-07-28T19:38:03.763 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:03 smithi138 bash[60615]: ts=2024-07-28T19:38:03.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:03.763 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:03 smithi138 bash[67894]: cluster 2024-07-28T19:38:02.732944+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-07-28T19:38:03.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:03 smithi100 bash[80199]: cluster 2024-07-28T19:38:02.732944+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-07-28T19:38:03.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:03 smithi100 bash[82644]: cluster 2024-07-28T19:38:02.732944+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-07-28T19:38:04.734 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:04 smithi138 bash[60615]: ts=2024-07-28T19:38:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:04.734 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:04 smithi138 bash[60615]: ts=2024-07-28T19:38:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:04.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:04 smithi100 bash[80199]: cluster 2024-07-28T19:38:03.594917+0000 mgr.y (mgr.25358) 101 : cluster [DBG] pgmap v192: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 19 MiB/s wr, 527 op/s 2024-07-28T19:38:04.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:04 smithi100 bash[80199]: cluster 2024-07-28T19:38:03.746652+0000 mon.a (mon.0) 335 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-07-28T19:38:04.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:04 smithi100 bash[82644]: cluster 2024-07-28T19:38:03.594917+0000 mgr.y (mgr.25358) 101 : cluster [DBG] pgmap v192: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 19 MiB/s wr, 527 op/s 2024-07-28T19:38:04.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:04 smithi100 bash[82644]: cluster 2024-07-28T19:38:03.746652+0000 mon.a (mon.0) 335 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-07-28T19:38:05.054 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:04 smithi138 bash[67894]: cluster 2024-07-28T19:38:03.594917+0000 mgr.y (mgr.25358) 101 : cluster [DBG] pgmap v192: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 19 MiB/s wr, 527 op/s 2024-07-28T19:38:05.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:04 smithi138 bash[67894]: cluster 2024-07-28T19:38:03.746652+0000 mon.a (mon.0) 335 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-07-28T19:38:06.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:05 smithi138 bash[67894]: cluster 2024-07-28T19:38:04.755045+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-07-28T19:38:06.185 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (4096 ms) 2024-07-28T19:38:06.186 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-07-28T19:38:06.244 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:05 smithi100 bash[82644]: cluster 2024-07-28T19:38:04.755045+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-07-28T19:38:06.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:05 smithi100 bash[80199]: cluster 2024-07-28T19:38:04.755045+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-07-28T19:38:07.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:06 smithi138 bash[67894]: cluster 2024-07-28T19:38:05.595603+0000 mgr.y (mgr.25358) 102 : cluster [DBG] pgmap v195: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 19 MiB/s wr, 528 op/s 2024-07-28T19:38:07.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:06 smithi138 bash[67894]: cluster 2024-07-28T19:38:05.759079+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-07-28T19:38:07.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:06 smithi100 bash[80199]: cluster 2024-07-28T19:38:05.595603+0000 mgr.y (mgr.25358) 102 : cluster [DBG] pgmap v195: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 19 MiB/s wr, 528 op/s 2024-07-28T19:38:07.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:06 smithi100 bash[80199]: cluster 2024-07-28T19:38:05.759079+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-07-28T19:38:07.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:06 smithi100 bash[82644]: cluster 2024-07-28T19:38:05.595603+0000 mgr.y (mgr.25358) 102 : cluster [DBG] pgmap v195: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 26 MiB/s rd, 19 MiB/s wr, 528 op/s 2024-07-28T19:38:07.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:06 smithi100 bash[82644]: cluster 2024-07-28T19:38:05.759079+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-07-28T19:38:08.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: cluster 2024-07-28T19:38:06.813220+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-07-28T19:38:08.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.558772+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.568798+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.570918+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:08.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.572969+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:38:08.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.583611+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.628113+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:08.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.674713+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:08.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.676148+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:08.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:07 smithi138 bash[67894]: audit 2024-07-28T19:38:07.677434+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: cluster 2024-07-28T19:38:06.813220+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.558772+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.568798+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.570918+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.572969+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.583611+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.628113+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.674713+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.676148+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:08.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[82644]: audit 2024-07-28T19:38:07.677434+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: cluster 2024-07-28T19:38:06.813220+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.558772+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.568798+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.570918+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.572969+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.583611+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.628113+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.674713+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.676148+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:07 smithi100 bash[80199]: audit 2024-07-28T19:38:07.677434+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:08.436 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2251 ms) 2024-07-28T19:38:08.436 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-07-28T19:38:09.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:08 smithi138 bash[60615]: ts=2024-07-28T19:38:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:09.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:08 smithi138 bash[67894]: cluster 2024-07-28T19:38:07.596227+0000 mgr.y (mgr.25358) 103 : cluster [DBG] pgmap v198: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:38:09.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:08 smithi138 bash[67894]: cephadm 2024-07-28T19:38:07.674061+0000 mgr.y (mgr.25358) 104 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-07-28T19:38:09.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:08 smithi138 bash[67894]: cephadm 2024-07-28T19:38:07.678777+0000 mgr.y (mgr.25358) 105 : cephadm [INF] Reconfiguring daemon mon.a on smithi100 2024-07-28T19:38:09.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:08 smithi138 bash[67894]: cluster 2024-07-28T19:38:07.826607+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-07-28T19:38:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:08 smithi100 bash[80199]: cluster 2024-07-28T19:38:07.596227+0000 mgr.y (mgr.25358) 103 : cluster [DBG] pgmap v198: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:38:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:08 smithi100 bash[80199]: cephadm 2024-07-28T19:38:07.674061+0000 mgr.y (mgr.25358) 104 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-07-28T19:38:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:08 smithi100 bash[80199]: cephadm 2024-07-28T19:38:07.678777+0000 mgr.y (mgr.25358) 105 : cephadm [INF] Reconfiguring daemon mon.a on smithi100 2024-07-28T19:38:09.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:08 smithi100 bash[80199]: cluster 2024-07-28T19:38:07.826607+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-07-28T19:38:09.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:08 smithi100 bash[82644]: cluster 2024-07-28T19:38:07.596227+0000 mgr.y (mgr.25358) 103 : cluster [DBG] pgmap v198: 105 pgs: 105 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-07-28T19:38:09.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:08 smithi100 bash[82644]: cephadm 2024-07-28T19:38:07.674061+0000 mgr.y (mgr.25358) 104 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-07-28T19:38:09.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:08 smithi100 bash[82644]: cephadm 2024-07-28T19:38:07.678777+0000 mgr.y (mgr.25358) 105 : cephadm [INF] Reconfiguring daemon mon.a on smithi100 2024-07-28T19:38:09.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:08 smithi100 bash[82644]: cluster 2024-07-28T19:38:07.826607+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-07-28T19:38:10.244 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:09 smithi100 bash[80199]: cluster 2024-07-28T19:38:08.835600+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-07-28T19:38:10.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:09 smithi100 bash[82644]: cluster 2024-07-28T19:38:08.835600+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-07-28T19:38:10.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:09 smithi138 bash[67894]: cluster 2024-07-28T19:38:08.835600+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-07-28T19:38:10.471 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2035 ms) 2024-07-28T19:38:10.472 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-07-28T19:38:10.513 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 38976996~4872260 2024-07-28T19:38:10.535 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 80258540~3627540 2024-07-28T19:38:10.535 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 73528486~5519765 2024-07-28T19:38:10.550 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 42626366~6277660 2024-07-28T19:38:10.552 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 45428694~5069657 2024-07-28T19:38:10.569 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 62213878~1308858 2024-07-28T19:38:10.570 INFO:tasks.workunit.client.0.smithi100.stdout:write: A 71801700~4391077 2024-07-28T19:38:10.585 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 79559878~1674368 2024-07-28T19:38:10.587 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap0 2024-07-28T19:38:10.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: cluster 2024-07-28T19:38:09.596903+0000 mgr.y (mgr.25358) 106 : cluster [DBG] pgmap v201: 105 pgs: 105 active+clean; 733 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 419 op/s 2024-07-28T19:38:10.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: cluster 2024-07-28T19:38:09.845514+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-07-28T19:38:10.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: audit 2024-07-28T19:38:10.131819+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:10.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: audit 2024-07-28T19:38:10.142120+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:10.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: cephadm 2024-07-28T19:38:10.144849+0000 mgr.y (mgr.25358) 107 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: audit 2024-07-28T19:38:10.145463+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: audit 2024-07-28T19:38:10.150296+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: audit 2024-07-28T19:38:10.153219+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[80199]: cephadm 2024-07-28T19:38:10.156248+0000 mgr.y (mgr.25358) 108 : cephadm [INF] Reconfiguring daemon mgr.y on smithi100 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: cluster 2024-07-28T19:38:09.596903+0000 mgr.y (mgr.25358) 106 : cluster [DBG] pgmap v201: 105 pgs: 105 active+clean; 733 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 419 op/s 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: cluster 2024-07-28T19:38:09.845514+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: audit 2024-07-28T19:38:10.131819+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: audit 2024-07-28T19:38:10.142120+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: cephadm 2024-07-28T19:38:10.144849+0000 mgr.y (mgr.25358) 107 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: audit 2024-07-28T19:38:10.145463+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: audit 2024-07-28T19:38:10.150296+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:38:10.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: audit 2024-07-28T19:38:10.153219+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:10.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:10 smithi100 bash[82644]: cephadm 2024-07-28T19:38:10.156248+0000 mgr.y (mgr.25358) 108 : cephadm [INF] Reconfiguring daemon mgr.y on smithi100 2024-07-28T19:38:11.171 INFO:tasks.workunit.client.0.smithi100.stdout:clone image281 -> image282 2024-07-28T19:38:11.292 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 155956053 2024-07-28T19:38:11.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: cluster 2024-07-28T19:38:09.596903+0000 mgr.y (mgr.25358) 106 : cluster [DBG] pgmap v201: 105 pgs: 105 active+clean; 733 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 419 op/s 2024-07-28T19:38:11.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: cluster 2024-07-28T19:38:09.845514+0000 mon.a (mon.0) 350 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-07-28T19:38:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: audit 2024-07-28T19:38:10.131819+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: audit 2024-07-28T19:38:10.142120+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: cephadm 2024-07-28T19:38:10.144849+0000 mgr.y (mgr.25358) 107 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-07-28T19:38:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: audit 2024-07-28T19:38:10.145463+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:38:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: audit 2024-07-28T19:38:10.150296+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:38:11.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: audit 2024-07-28T19:38:10.153219+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:11.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:10 smithi138 bash[67894]: cephadm 2024-07-28T19:38:10.156248+0000 mgr.y (mgr.25358) 108 : cephadm [INF] Reconfiguring daemon mgr.y on smithi100 2024-07-28T19:38:11.307 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 127184170~976333 2024-07-28T19:38:11.325 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 120343052~1034420 2024-07-28T19:38:11.326 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 120489951~3335908 2024-07-28T19:38:11.346 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 139956763~1261564 2024-07-28T19:38:11.347 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 28374830~5627776 2024-07-28T19:38:11.380 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 62527147~2107416 2024-07-28T19:38:11.388 INFO:tasks.workunit.client.0.smithi100.stdout:write: B 124157441~4293489 2024-07-28T19:38:11.406 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 20873526~579433 2024-07-28T19:38:11.414 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:38:12.189 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:12 smithi100 bash[80199]: cluster 2024-07-28T19:38:11.145531+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-07-28T19:38:12.189 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:12 smithi100 bash[82644]: cluster 2024-07-28T19:38:11.145531+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-07-28T19:38:12.192 INFO:tasks.workunit.client.0.smithi100.stdout:clone image282 -> image283 2024-07-28T19:38:12.354 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 221983060 2024-07-28T19:38:12.369 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 185610436~5344465 2024-07-28T19:38:12.402 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 191832803~2043518 2024-07-28T19:38:12.407 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 132263715~7173726 2024-07-28T19:38:12.442 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 1719694~2957551 2024-07-28T19:38:12.450 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 67343325~2108393 2024-07-28T19:38:12.466 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 131539786~3381928 2024-07-28T19:38:12.472 INFO:tasks.workunit.client.0.smithi100.stdout:write: C 49747238~7709090 2024-07-28T19:38:12.515 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 96297569~8103923 2024-07-28T19:38:12.523 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:38:12.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:12 smithi138 bash[67894]: cluster 2024-07-28T19:38:11.145531+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-07-28T19:38:12.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:12 smithi138 bash[60615]: ts=2024-07-28T19:38:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:13.143 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:38:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:38:12] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.43.0" 2024-07-28T19:38:13.189 INFO:tasks.workunit.client.0.smithi100.stdout:clone image283 -> image284 2024-07-28T19:38:13.385 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 89218939~2764029 2024-07-28T19:38:13.416 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 143432758~6138362 2024-07-28T19:38:13.426 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 53421903~6810172 2024-07-28T19:38:13.472 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 137550229~6478747 2024-07-28T19:38:13.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[80199]: cluster 2024-07-28T19:38:11.597573+0000 mgr.y (mgr.25358) 109 : cluster [DBG] pgmap v204: 105 pgs: 105 active+clean; 733 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 419 op/s 2024-07-28T19:38:13.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[80199]: cluster 2024-07-28T19:38:12.157234+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-07-28T19:38:13.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[80199]: audit 2024-07-28T19:38:12.755399+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:13.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[80199]: audit 2024-07-28T19:38:12.765135+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:13.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[80199]: audit 2024-07-28T19:38:12.767406+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:13.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[80199]: audit 2024-07-28T19:38:12.768901+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:13.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[80199]: audit 2024-07-28T19:38:12.770088+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:13.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[82644]: cluster 2024-07-28T19:38:11.597573+0000 mgr.y (mgr.25358) 109 : cluster [DBG] pgmap v204: 105 pgs: 105 active+clean; 733 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 419 op/s 2024-07-28T19:38:13.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[82644]: cluster 2024-07-28T19:38:12.157234+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-07-28T19:38:13.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[82644]: audit 2024-07-28T19:38:12.755399+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:13.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[82644]: audit 2024-07-28T19:38:12.765135+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:13.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[82644]: audit 2024-07-28T19:38:12.767406+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:13.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[82644]: audit 2024-07-28T19:38:12.768901+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:13.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:13 smithi100 bash[82644]: audit 2024-07-28T19:38:12.770088+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:13.499 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 135497094~3669688 2024-07-28T19:38:13.505 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:13 smithi138 bash[67894]: cluster 2024-07-28T19:38:11.597573+0000 mgr.y (mgr.25358) 109 : cluster [DBG] pgmap v204: 105 pgs: 105 active+clean; 733 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 15 MiB/s wr, 419 op/s 2024-07-28T19:38:13.505 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:13 smithi138 bash[67894]: cluster 2024-07-28T19:38:12.157234+0000 mon.a (mon.0) 357 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-07-28T19:38:13.505 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:13 smithi138 bash[67894]: audit 2024-07-28T19:38:12.755399+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:13.505 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:13 smithi138 bash[67894]: audit 2024-07-28T19:38:12.765135+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:13.506 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:13 smithi138 bash[67894]: audit 2024-07-28T19:38:12.767406+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:13.506 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:13 smithi138 bash[67894]: audit 2024-07-28T19:38:12.768901+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:13.506 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:13 smithi138 bash[67894]: audit 2024-07-28T19:38:12.770088+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:13.518 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 37724880~2224075 2024-07-28T19:38:13.554 INFO:tasks.workunit.client.0.smithi100.stdout:write: D 90095784~5705422 2024-07-28T19:38:13.586 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 144487084~5960240 2024-07-28T19:38:13.589 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap3 2024-07-28T19:38:13.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:13 smithi138 bash[60615]: ts=2024-07-28T19:38:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:14.209 INFO:tasks.workunit.client.0.smithi100.stdout:clone image284 -> image285 2024-07-28T19:38:14.366 INFO:tasks.workunit.client.0.smithi100.stdout:resize: 138653819 2024-07-28T19:38:14.436 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:14 smithi100 bash[80199]: cephadm 2024-07-28T19:38:12.766849+0000 mgr.y (mgr.25358) 110 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-28T19:38:14.437 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:14 smithi100 bash[80199]: cephadm 2024-07-28T19:38:12.771642+0000 mgr.y (mgr.25358) 111 : cephadm [INF] Reconfiguring daemon mon.c on smithi100 2024-07-28T19:38:14.437 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:14 smithi100 bash[80199]: cluster 2024-07-28T19:38:13.160311+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-07-28T19:38:14.437 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:14 smithi100 bash[82644]: cephadm 2024-07-28T19:38:12.766849+0000 mgr.y (mgr.25358) 110 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-28T19:38:14.437 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:14 smithi100 bash[82644]: cephadm 2024-07-28T19:38:12.771642+0000 mgr.y (mgr.25358) 111 : cephadm [INF] Reconfiguring daemon mon.c on smithi100 2024-07-28T19:38:14.437 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:14 smithi100 bash[82644]: cluster 2024-07-28T19:38:13.160311+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-07-28T19:38:14.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:14 smithi138 bash[67894]: cephadm 2024-07-28T19:38:12.766849+0000 mgr.y (mgr.25358) 110 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-07-28T19:38:14.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:14 smithi138 bash[67894]: cephadm 2024-07-28T19:38:12.771642+0000 mgr.y (mgr.25358) 111 : cephadm [INF] Reconfiguring daemon mon.c on smithi100 2024-07-28T19:38:14.451 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:14 smithi138 bash[67894]: cluster 2024-07-28T19:38:13.160311+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-07-28T19:38:14.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:14 smithi138 bash[60615]: ts=2024-07-28T19:38:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:14.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:14 smithi138 bash[60615]: ts=2024-07-28T19:38:14.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:14.901 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (4430 ms) 2024-07-28T19:38:14.901 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-07-28T19:38:14.945 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:15.113 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (212 ms) 2024-07-28T19:38:15.113 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-07-28T19:38:15.165 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:15.271 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (158 ms) 2024-07-28T19:38:15.271 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-07-28T19:38:15.316 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:15.471 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Clone_StrippingLargerDstObjSize (200 ms) 2024-07-28T19:38:15.472 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-07-28T19:38:15.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[80199]: cluster 2024-07-28T19:38:13.598858+0000 mgr.y (mgr.25358) 112 : cluster [DBG] pgmap v207: 105 pgs: 105 active+clean; 933 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 44 MiB/s wr, 740 op/s 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[80199]: cluster 2024-07-28T19:38:14.179878+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[80199]: audit 2024-07-28T19:38:15.154803+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[80199]: audit 2024-07-28T19:38:15.160926+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[80199]: audit 2024-07-28T19:38:15.162322+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[80199]: audit 2024-07-28T19:38:15.163114+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[82644]: cluster 2024-07-28T19:38:13.598858+0000 mgr.y (mgr.25358) 112 : cluster [DBG] pgmap v207: 105 pgs: 105 active+clean; 933 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 44 MiB/s wr, 740 op/s 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[82644]: cluster 2024-07-28T19:38:14.179878+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[82644]: audit 2024-07-28T19:38:15.154803+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[82644]: audit 2024-07-28T19:38:15.160926+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[82644]: audit 2024-07-28T19:38:15.162322+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:38:15.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:15 smithi100 bash[82644]: audit 2024-07-28T19:38:15.163114+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:15.505 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:15 smithi138 bash[67894]: cluster 2024-07-28T19:38:13.598858+0000 mgr.y (mgr.25358) 112 : cluster [DBG] pgmap v207: 105 pgs: 105 active+clean; 933 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 44 MiB/s wr, 740 op/s 2024-07-28T19:38:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:15 smithi138 bash[67894]: cluster 2024-07-28T19:38:14.179878+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-07-28T19:38:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:15 smithi138 bash[67894]: audit 2024-07-28T19:38:15.154803+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:15 smithi138 bash[67894]: audit 2024-07-28T19:38:15.160926+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:15 smithi138 bash[67894]: audit 2024-07-28T19:38:15.162322+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:38:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:15 smithi138 bash[67894]: audit 2024-07-28T19:38:15.163114+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:15.645 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (174 ms) 2024-07-28T19:38:15.646 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-07-28T19:38:15.702 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:15.874 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Stress_StrippingLargerDstObjSize (229 ms) 2024-07-28T19:38:15.874 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-07-28T19:38:15.931 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:16.102 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (228 ms) 2024-07-28T19:38:16.102 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-07-28T19:38:16.163 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:16.330 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (228 ms) 2024-07-28T19:38:16.330 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-07-28T19:38:16.381 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:16.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:16 smithi100 bash[80199]: cephadm 2024-07-28T19:38:15.161989+0000 mgr.y (mgr.25358) 113 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-07-28T19:38:16.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:16 smithi100 bash[80199]: cephadm 2024-07-28T19:38:15.165898+0000 mgr.y (mgr.25358) 114 : cephadm [INF] Reconfiguring daemon osd.0 on smithi100 2024-07-28T19:38:16.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:16 smithi100 bash[82644]: cephadm 2024-07-28T19:38:15.161989+0000 mgr.y (mgr.25358) 113 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-07-28T19:38:16.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:16 smithi100 bash[82644]: cephadm 2024-07-28T19:38:15.165898+0000 mgr.y (mgr.25358) 114 : cephadm [INF] Reconfiguring daemon osd.0 on smithi100 2024-07-28T19:38:16.530 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (200 ms) 2024-07-28T19:38:16.531 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-07-28T19:38:16.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:16 smithi138 bash[67894]: cephadm 2024-07-28T19:38:15.161989+0000 mgr.y (mgr.25358) 113 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-07-28T19:38:16.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:16 smithi138 bash[67894]: cephadm 2024-07-28T19:38:15.165898+0000 mgr.y (mgr.25358) 114 : cephadm [INF] Reconfiguring daemon osd.0 on smithi100 2024-07-28T19:38:16.582 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:16.718 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (188 ms) 2024-07-28T19:38:16.718 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-07-28T19:38:16.744 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:16.840 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (122 ms) 2024-07-28T19:38:17.192 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:17 smithi100 bash[80199]: cluster 2024-07-28T19:38:15.599604+0000 mgr.y (mgr.25358) 115 : cluster [DBG] pgmap v209: 105 pgs: 105 active+clean; 933 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 32 MiB/s rd, 40 MiB/s wr, 664 op/s 2024-07-28T19:38:17.195 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 31 tests from TestDeepCopy (70029 ms total) 2024-07-28T19:38:17.196 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:38:17.196 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 51 tests from TestMirroring 2024-07-28T19:38:17.494 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:17 smithi100 bash[82644]: cluster 2024-07-28T19:38:15.599604+0000 mgr.y (mgr.25358) 115 : cluster [DBG] pgmap v209: 105 pgs: 105 active+clean; 933 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 32 MiB/s rd, 40 MiB/s wr, 664 op/s 2024-07-28T19:38:17.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:17 smithi138 bash[67894]: cluster 2024-07-28T19:38:15.599604+0000 mgr.y (mgr.25358) 115 : cluster [DBG] pgmap v209: 105 pgs: 105 active+clean; 933 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 32 MiB/s rd, 40 MiB/s wr, 664 op/s 2024-07-28T19:38:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[82644]: cluster 2024-07-28T19:38:17.187009+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-07-28T19:38:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[82644]: audit 2024-07-28T19:38:17.684091+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[82644]: audit 2024-07-28T19:38:17.695239+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[82644]: audit 2024-07-28T19:38:17.697135+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:38:18.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[82644]: audit 2024-07-28T19:38:17.698188+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:18.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[80199]: cluster 2024-07-28T19:38:17.187009+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-07-28T19:38:18.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[80199]: audit 2024-07-28T19:38:17.684091+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:18.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[80199]: audit 2024-07-28T19:38:17.695239+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:18.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[80199]: audit 2024-07-28T19:38:17.697135+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:38:18.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:18 smithi100 bash[80199]: audit 2024-07-28T19:38:17.698188+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:18.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:18 smithi138 bash[67894]: cluster 2024-07-28T19:38:17.187009+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-07-28T19:38:18.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:18 smithi138 bash[67894]: audit 2024-07-28T19:38:17.684091+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:18.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:18 smithi138 bash[67894]: audit 2024-07-28T19:38:17.695239+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:18.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:18 smithi138 bash[67894]: audit 2024-07-28T19:38:17.697135+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:38:18.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:18 smithi138 bash[67894]: audit 2024-07-28T19:38:17.698188+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:19.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:18 smithi138 bash[60615]: ts=2024-07-28T19:38:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:19.207 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-07-28T19:38:19.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[80199]: cluster 2024-07-28T19:38:17.600005+0000 mgr.y (mgr.25358) 116 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:38:19.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[80199]: cephadm 2024-07-28T19:38:17.696833+0000 mgr.y (mgr.25358) 117 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-07-28T19:38:19.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[80199]: cephadm 2024-07-28T19:38:17.701937+0000 mgr.y (mgr.25358) 118 : cephadm [INF] Reconfiguring daemon osd.1 on smithi100 2024-07-28T19:38:19.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[80199]: cluster 2024-07-28T19:38:18.216387+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-07-28T19:38:19.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[80199]: audit 2024-07-28T19:38:18.219821+0000 mon.a (mon.0) 375 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:38:19.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[82644]: cluster 2024-07-28T19:38:17.600005+0000 mgr.y (mgr.25358) 116 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:38:19.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[82644]: cephadm 2024-07-28T19:38:17.696833+0000 mgr.y (mgr.25358) 117 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-07-28T19:38:19.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[82644]: cephadm 2024-07-28T19:38:17.701937+0000 mgr.y (mgr.25358) 118 : cephadm [INF] Reconfiguring daemon osd.1 on smithi100 2024-07-28T19:38:19.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[82644]: cluster 2024-07-28T19:38:18.216387+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-07-28T19:38:19.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:19 smithi100 bash[82644]: audit 2024-07-28T19:38:18.219821+0000 mon.a (mon.0) 375 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:38:19.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:19 smithi138 bash[67894]: cluster 2024-07-28T19:38:17.600005+0000 mgr.y (mgr.25358) 116 : cluster [DBG] pgmap v211: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-07-28T19:38:19.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:19 smithi138 bash[67894]: cephadm 2024-07-28T19:38:17.696833+0000 mgr.y (mgr.25358) 117 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-07-28T19:38:19.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:19 smithi138 bash[67894]: cephadm 2024-07-28T19:38:17.701937+0000 mgr.y (mgr.25358) 118 : cephadm [INF] Reconfiguring daemon osd.1 on smithi100 2024-07-28T19:38:19.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:19 smithi138 bash[67894]: cluster 2024-07-28T19:38:18.216387+0000 mon.a (mon.0) 374 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-07-28T19:38:19.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:19 smithi138 bash[67894]: audit 2024-07-28T19:38:18.219821+0000 mon.a (mon.0) 375 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:38:20.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[80199]: audit 2024-07-28T19:38:19.211863+0000 mon.a (mon.0) 376 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:38:20.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[80199]: cluster 2024-07-28T19:38:19.212171+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-07-28T19:38:20.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[80199]: audit 2024-07-28T19:38:20.125169+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:20.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[80199]: audit 2024-07-28T19:38:20.134720+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:20.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[80199]: audit 2024-07-28T19:38:20.136994+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:38:20.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[80199]: audit 2024-07-28T19:38:20.138471+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:20.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[80199]: cluster 2024-07-28T19:38:20.220929+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-07-28T19:38:20.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[82644]: audit 2024-07-28T19:38:19.211863+0000 mon.a (mon.0) 376 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:38:20.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[82644]: cluster 2024-07-28T19:38:19.212171+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-07-28T19:38:20.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[82644]: audit 2024-07-28T19:38:20.125169+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:20.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[82644]: audit 2024-07-28T19:38:20.134720+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:20.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[82644]: audit 2024-07-28T19:38:20.136994+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:38:20.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[82644]: audit 2024-07-28T19:38:20.138471+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:20.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:20 smithi100 bash[82644]: cluster 2024-07-28T19:38:20.220929+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-07-28T19:38:20.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:20 smithi138 bash[67894]: audit 2024-07-28T19:38:19.211863+0000 mon.a (mon.0) 376 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:38:20.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:20 smithi138 bash[67894]: cluster 2024-07-28T19:38:19.212171+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-07-28T19:38:20.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:20 smithi138 bash[67894]: audit 2024-07-28T19:38:20.125169+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:20.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:20 smithi138 bash[67894]: audit 2024-07-28T19:38:20.134720+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:20.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:20 smithi138 bash[67894]: audit 2024-07-28T19:38:20.136994+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:38:20.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:20 smithi138 bash[67894]: audit 2024-07-28T19:38:20.138471+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:20.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:20 smithi138 bash[67894]: cluster 2024-07-28T19:38:20.220929+0000 mon.a (mon.0) 382 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-07-28T19:38:21.407 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2200 ms) 2024-07-28T19:38:21.407 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-07-28T19:38:21.494 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:21 smithi100 bash[80199]: cluster 2024-07-28T19:38:19.600785+0000 mgr.y (mgr.25358) 119 : cluster [DBG] pgmap v214: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:38:21.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:21 smithi100 bash[80199]: cephadm 2024-07-28T19:38:20.136488+0000 mgr.y (mgr.25358) 120 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-07-28T19:38:21.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:21 smithi100 bash[80199]: cephadm 2024-07-28T19:38:20.141540+0000 mgr.y (mgr.25358) 121 : cephadm [INF] Reconfiguring daemon osd.2 on smithi100 2024-07-28T19:38:21.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:21 smithi100 bash[82644]: cluster 2024-07-28T19:38:19.600785+0000 mgr.y (mgr.25358) 119 : cluster [DBG] pgmap v214: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:38:21.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:21 smithi100 bash[82644]: cephadm 2024-07-28T19:38:20.136488+0000 mgr.y (mgr.25358) 120 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-07-28T19:38:21.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:21 smithi100 bash[82644]: cephadm 2024-07-28T19:38:20.141540+0000 mgr.y (mgr.25358) 121 : cephadm [INF] Reconfiguring daemon osd.2 on smithi100 2024-07-28T19:38:21.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:21 smithi138 bash[67894]: cluster 2024-07-28T19:38:19.600785+0000 mgr.y (mgr.25358) 119 : cluster [DBG] pgmap v214: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:38:21.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:21 smithi138 bash[67894]: cephadm 2024-07-28T19:38:20.136488+0000 mgr.y (mgr.25358) 120 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-07-28T19:38:21.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:21 smithi138 bash[67894]: cephadm 2024-07-28T19:38:20.141540+0000 mgr.y (mgr.25358) 121 : cephadm [INF] Reconfiguring daemon osd.2 on smithi100 2024-07-28T19:38:21.579 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (172 ms) 2024-07-28T19:38:21.580 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-07-28T19:38:21.710 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (131 ms) 2024-07-28T19:38:21.710 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-07-28T19:38:21.922 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (211 ms) 2024-07-28T19:38:21.922 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-07-28T19:38:22.086 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (164 ms) 2024-07-28T19:38:22.086 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-07-28T19:38:22.248 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (162 ms) 2024-07-28T19:38:22.249 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-07-28T19:38:22.439 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (191 ms) 2024-07-28T19:38:22.439 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-07-28T19:38:22.440 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:22.440 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-07-28T19:38:22.440 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-07-28T19:38:22.440 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:38:22.440 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-07-28T19:38:22.440 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-07-28T19:38:22.551 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (112 ms) 2024-07-28T19:38:22.551 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-07-28T19:38:22.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:22 smithi138 bash[67894]: cluster 2024-07-28T19:38:21.238198+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-07-28T19:38:22.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:22 smithi138 bash[60615]: ts=2024-07-28T19:38:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:22.683 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (132 ms) 2024-07-28T19:38:22.683 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-07-28T19:38:22.744 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:22 smithi100 bash[80199]: cluster 2024-07-28T19:38:21.238198+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-07-28T19:38:22.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:22 smithi100 bash[82644]: cluster 2024-07-28T19:38:21.238198+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-07-28T19:38:23.244 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:38:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:38:22] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2024-07-28T19:38:23.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:23 smithi138 bash[67894]: cluster 2024-07-28T19:38:21.601290+0000 mgr.y (mgr.25358) 122 : cluster [DBG] pgmap v217: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:38:23.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:23 smithi138 bash[67894]: audit 2024-07-28T19:38:22.484549+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:23.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:23 smithi138 bash[67894]: audit 2024-07-28T19:38:22.491272+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:23.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:23 smithi138 bash[67894]: audit 2024-07-28T19:38:22.492802+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:38:23.509 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:23 smithi138 bash[67894]: audit 2024-07-28T19:38:22.493758+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:23.510 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:23 smithi138 bash[67894]: audit 2024-07-28T19:38:22.628206+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:23.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[80199]: cluster 2024-07-28T19:38:21.601290+0000 mgr.y (mgr.25358) 122 : cluster [DBG] pgmap v217: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:38:23.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[80199]: audit 2024-07-28T19:38:22.484549+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:23.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[80199]: audit 2024-07-28T19:38:22.491272+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:23.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[80199]: audit 2024-07-28T19:38:22.492802+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:38:23.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[80199]: audit 2024-07-28T19:38:22.493758+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:23.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[80199]: audit 2024-07-28T19:38:22.628206+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:23.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[82644]: cluster 2024-07-28T19:38:21.601290+0000 mgr.y (mgr.25358) 122 : cluster [DBG] pgmap v217: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-07-28T19:38:23.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[82644]: audit 2024-07-28T19:38:22.484549+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:23.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[82644]: audit 2024-07-28T19:38:22.491272+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:23.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[82644]: audit 2024-07-28T19:38:22.492802+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:38:23.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[82644]: audit 2024-07-28T19:38:22.493758+0000 mon.a (mon.0) 387 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:23.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:23 smithi100 bash[82644]: audit 2024-07-28T19:38:22.628206+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:23.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:23 smithi138 bash[60615]: ts=2024-07-28T19:38:23.511Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:24.386 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:24 smithi100 bash[80199]: cephadm 2024-07-28T19:38:22.492406+0000 mgr.y (mgr.25358) 123 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-07-28T19:38:24.386 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:24 smithi100 bash[80199]: cephadm 2024-07-28T19:38:22.496001+0000 mgr.y (mgr.25358) 124 : cephadm [INF] Reconfiguring daemon osd.3 on smithi100 2024-07-28T19:38:24.387 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:24 smithi100 bash[80199]: cluster 2024-07-28T19:38:23.496703+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-07-28T19:38:24.387 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:24 smithi100 bash[82644]: cephadm 2024-07-28T19:38:22.492406+0000 mgr.y (mgr.25358) 123 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-07-28T19:38:24.387 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:24 smithi100 bash[82644]: cephadm 2024-07-28T19:38:22.496001+0000 mgr.y (mgr.25358) 124 : cephadm [INF] Reconfiguring daemon osd.3 on smithi100 2024-07-28T19:38:24.387 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:24 smithi100 bash[82644]: cluster 2024-07-28T19:38:23.496703+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-07-28T19:38:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:24 smithi138 bash[67894]: cephadm 2024-07-28T19:38:22.492406+0000 mgr.y (mgr.25358) 123 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-07-28T19:38:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:24 smithi138 bash[67894]: cephadm 2024-07-28T19:38:22.496001+0000 mgr.y (mgr.25358) 124 : cephadm [INF] Reconfiguring daemon osd.3 on smithi100 2024-07-28T19:38:24.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:24 smithi138 bash[67894]: cluster 2024-07-28T19:38:23.496703+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-07-28T19:38:24.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:24 smithi138 bash[60615]: ts=2024-07-28T19:38:24.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:24.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:24 smithi138 bash[60615]: ts=2024-07-28T19:38:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:24.609 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (1926 ms) 2024-07-28T19:38:24.609 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-07-28T19:38:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:25 smithi138 bash[67894]: cluster 2024-07-28T19:38:23.602785+0000 mgr.y (mgr.25358) 125 : cluster [DBG] pgmap v219: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 340 KiB/s rd, 47 KiB/s wr, 492 op/s 2024-07-28T19:38:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:25 smithi138 bash[67894]: cluster 2024-07-28T19:38:24.507994+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-07-28T19:38:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:25 smithi138 bash[67894]: cluster 2024-07-28T19:38:24.814461+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-07-28T19:38:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:25 smithi138 bash[67894]: audit 2024-07-28T19:38:25.003755+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:25.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:25 smithi138 bash[67894]: audit 2024-07-28T19:38:25.009284+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:25 smithi138 bash[67894]: audit 2024-07-28T19:38:25.010947+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:38:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:25 smithi138 bash[67894]: audit 2024-07-28T19:38:25.011933+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[82644]: cluster 2024-07-28T19:38:23.602785+0000 mgr.y (mgr.25358) 125 : cluster [DBG] pgmap v219: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 340 KiB/s rd, 47 KiB/s wr, 492 op/s 2024-07-28T19:38:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[82644]: cluster 2024-07-28T19:38:24.507994+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-07-28T19:38:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[82644]: cluster 2024-07-28T19:38:24.814461+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-07-28T19:38:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[82644]: audit 2024-07-28T19:38:25.003755+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[82644]: audit 2024-07-28T19:38:25.009284+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[82644]: audit 2024-07-28T19:38:25.010947+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:38:25.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[82644]: audit 2024-07-28T19:38:25.011933+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:25.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[80199]: cluster 2024-07-28T19:38:23.602785+0000 mgr.y (mgr.25358) 125 : cluster [DBG] pgmap v219: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 340 KiB/s rd, 47 KiB/s wr, 492 op/s 2024-07-28T19:38:25.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[80199]: cluster 2024-07-28T19:38:24.507994+0000 mon.a (mon.0) 390 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-07-28T19:38:25.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[80199]: cluster 2024-07-28T19:38:24.814461+0000 mon.a (mon.0) 391 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-07-28T19:38:25.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[80199]: audit 2024-07-28T19:38:25.003755+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:25.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[80199]: audit 2024-07-28T19:38:25.009284+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:25.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[80199]: audit 2024-07-28T19:38:25.010947+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi100.ppcqpk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:38:25.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:25 smithi100 bash[80199]: audit 2024-07-28T19:38:25.011933+0000 mon.a (mon.0) 395 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:25.898 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (1289 ms) 2024-07-28T19:38:25.899 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-07-28T19:38:26.122 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (224 ms) 2024-07-28T19:38:26.122 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-07-28T19:38:26.340 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (217 ms) 2024-07-28T19:38:26.340 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-07-28T19:38:26.548 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (209 ms) 2024-07-28T19:38:26.548 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-07-28T19:38:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:26 smithi138 bash[67894]: cephadm 2024-07-28T19:38:25.010630+0000 mgr.y (mgr.25358) 126 : cephadm [INF] Reconfiguring mds.foo.smithi100.ppcqpk (monmap changed)... 2024-07-28T19:38:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:26 smithi138 bash[67894]: cephadm 2024-07-28T19:38:25.012863+0000 mgr.y (mgr.25358) 127 : cephadm [INF] Reconfiguring daemon mds.foo.smithi100.ppcqpk on smithi100 2024-07-28T19:38:26.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:26 smithi138 bash[67894]: cluster 2024-07-28T19:38:25.813956+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-07-28T19:38:26.576 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:26 smithi100 bash[80199]: cephadm 2024-07-28T19:38:25.010630+0000 mgr.y (mgr.25358) 126 : cephadm [INF] Reconfiguring mds.foo.smithi100.ppcqpk (monmap changed)... 2024-07-28T19:38:26.576 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:26 smithi100 bash[80199]: cephadm 2024-07-28T19:38:25.012863+0000 mgr.y (mgr.25358) 127 : cephadm [INF] Reconfiguring daemon mds.foo.smithi100.ppcqpk on smithi100 2024-07-28T19:38:26.576 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:26 smithi100 bash[80199]: cluster 2024-07-28T19:38:25.813956+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-07-28T19:38:26.576 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:26 smithi100 bash[82644]: cephadm 2024-07-28T19:38:25.010630+0000 mgr.y (mgr.25358) 126 : cephadm [INF] Reconfiguring mds.foo.smithi100.ppcqpk (monmap changed)... 2024-07-28T19:38:26.577 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:26 smithi100 bash[82644]: cephadm 2024-07-28T19:38:25.012863+0000 mgr.y (mgr.25358) 127 : cephadm [INF] Reconfiguring daemon mds.foo.smithi100.ppcqpk on smithi100 2024-07-28T19:38:26.577 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:26 smithi100 bash[82644]: cluster 2024-07-28T19:38:25.813956+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-07-28T19:38:26.677 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (129 ms) 2024-07-28T19:38:26.678 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-07-28T19:38:26.811 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (134 ms) 2024-07-28T19:38:26.811 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-07-28T19:38:26.997 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (186 ms) 2024-07-28T19:38:26.997 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-07-28T19:38:27.252 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (254 ms) 2024-07-28T19:38:27.252 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-07-28T19:38:27.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:27 smithi138 bash[67894]: cluster 2024-07-28T19:38:25.603520+0000 mgr.y (mgr.25358) 128 : cluster [DBG] pgmap v222: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 47 KiB/s wr, 494 op/s 2024-07-28T19:38:27.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:27 smithi100 bash[80199]: cluster 2024-07-28T19:38:25.603520+0000 mgr.y (mgr.25358) 128 : cluster [DBG] pgmap v222: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 47 KiB/s wr, 494 op/s 2024-07-28T19:38:27.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:27 smithi100 bash[82644]: cluster 2024-07-28T19:38:25.603520+0000 mgr.y (mgr.25358) 128 : cluster [DBG] pgmap v222: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 47 KiB/s wr, 494 op/s 2024-07-28T19:38:28.657 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:28 smithi138 bash[67894]: audit 2024-07-28T19:38:27.339833+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:28.657 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:28 smithi138 bash[67894]: audit 2024-07-28T19:38:27.347087+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:28.658 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:28 smithi138 bash[67894]: audit 2024-07-28T19:38:27.348970+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:28.658 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:28 smithi138 bash[67894]: audit 2024-07-28T19:38:27.350104+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:28.658 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:28 smithi138 bash[67894]: audit 2024-07-28T19:38:27.351023+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:28.658 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:28 smithi138 bash[67894]: cluster 2024-07-28T19:38:28.281163+0000 mon.a (mon.0) 402 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-07-28T19:38:28.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[80199]: audit 2024-07-28T19:38:27.339833+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:28.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[80199]: audit 2024-07-28T19:38:27.347087+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:28.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[80199]: audit 2024-07-28T19:38:27.348970+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:28.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[80199]: audit 2024-07-28T19:38:27.350104+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:28.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[80199]: audit 2024-07-28T19:38:27.351023+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:28.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[80199]: cluster 2024-07-28T19:38:28.281163+0000 mon.a (mon.0) 402 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-07-28T19:38:28.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[82644]: audit 2024-07-28T19:38:27.339833+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:28.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[82644]: audit 2024-07-28T19:38:27.347087+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:28.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[82644]: audit 2024-07-28T19:38:27.348970+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-07-28T19:38:28.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[82644]: audit 2024-07-28T19:38:27.350104+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-07-28T19:38:28.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[82644]: audit 2024-07-28T19:38:27.351023+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:28.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:28 smithi100 bash[82644]: cluster 2024-07-28T19:38:28.281163+0000 mon.a (mon.0) 402 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-07-28T19:38:29.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:28 smithi138 bash[60615]: ts=2024-07-28T19:38:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:29.447 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (2196 ms) 2024-07-28T19:38:29.447 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-07-28T19:38:29.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:29 smithi138 bash[67894]: cephadm 2024-07-28T19:38:27.348600+0000 mgr.y (mgr.25358) 129 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-28T19:38:29.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:29 smithi138 bash[67894]: cephadm 2024-07-28T19:38:27.352262+0000 mgr.y (mgr.25358) 130 : cephadm [INF] Reconfiguring daemon mon.b on smithi138 2024-07-28T19:38:29.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:29 smithi138 bash[67894]: cluster 2024-07-28T19:38:27.604230+0000 mgr.y (mgr.25358) 131 : cluster [DBG] pgmap v224: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 363 KiB/s rd, 50 KiB/s wr, 525 op/s 2024-07-28T19:38:29.659 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (212 ms) 2024-07-28T19:38:29.659 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-07-28T19:38:29.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:29 smithi100 bash[80199]: cephadm 2024-07-28T19:38:27.348600+0000 mgr.y (mgr.25358) 129 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-28T19:38:29.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:29 smithi100 bash[80199]: cephadm 2024-07-28T19:38:27.352262+0000 mgr.y (mgr.25358) 130 : cephadm [INF] Reconfiguring daemon mon.b on smithi138 2024-07-28T19:38:29.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:29 smithi100 bash[80199]: cluster 2024-07-28T19:38:27.604230+0000 mgr.y (mgr.25358) 131 : cluster [DBG] pgmap v224: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 363 KiB/s rd, 50 KiB/s wr, 525 op/s 2024-07-28T19:38:29.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:29 smithi100 bash[82644]: cephadm 2024-07-28T19:38:27.348600+0000 mgr.y (mgr.25358) 129 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-07-28T19:38:29.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:29 smithi100 bash[82644]: cephadm 2024-07-28T19:38:27.352262+0000 mgr.y (mgr.25358) 130 : cephadm [INF] Reconfiguring daemon mon.b on smithi138 2024-07-28T19:38:29.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:29 smithi100 bash[82644]: cluster 2024-07-28T19:38:27.604230+0000 mgr.y (mgr.25358) 131 : cluster [DBG] pgmap v224: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 363 KiB/s rd, 50 KiB/s wr, 525 op/s 2024-07-28T19:38:29.901 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (241 ms) 2024-07-28T19:38:29.901 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-07-28T19:38:30.129 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (228 ms) 2024-07-28T19:38:30.129 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-07-28T19:38:30.460 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (332 ms) 2024-07-28T19:38:30.461 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-07-28T19:38:30.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: cluster 2024-07-28T19:38:29.359959+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-07-28T19:38:30.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: cluster 2024-07-28T19:38:29.604768+0000 mgr.y (mgr.25358) 132 : cluster [DBG] pgmap v227: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 39 KiB/s wr, 468 op/s 2024-07-28T19:38:30.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: audit 2024-07-28T19:38:30.158348+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:30.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: audit 2024-07-28T19:38:30.168130+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:30.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: cephadm 2024-07-28T19:38:30.170016+0000 mgr.y (mgr.25358) 133 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-07-28T19:38:30.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: audit 2024-07-28T19:38:30.170724+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: audit 2024-07-28T19:38:30.172585+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: audit 2024-07-28T19:38:30.174034+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[80199]: cephadm 2024-07-28T19:38:30.175574+0000 mgr.y (mgr.25358) 134 : cephadm [INF] Reconfiguring daemon mgr.x on smithi138 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: cluster 2024-07-28T19:38:29.359959+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: cluster 2024-07-28T19:38:29.604768+0000 mgr.y (mgr.25358) 132 : cluster [DBG] pgmap v227: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 39 KiB/s wr, 468 op/s 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: audit 2024-07-28T19:38:30.158348+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: audit 2024-07-28T19:38:30.168130+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: cephadm 2024-07-28T19:38:30.170016+0000 mgr.y (mgr.25358) 133 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-07-28T19:38:30.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: audit 2024-07-28T19:38:30.170724+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:38:30.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: audit 2024-07-28T19:38:30.172585+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:38:30.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: audit 2024-07-28T19:38:30.174034+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:30.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:30 smithi100 bash[82644]: cephadm 2024-07-28T19:38:30.175574+0000 mgr.y (mgr.25358) 134 : cephadm [INF] Reconfiguring daemon mgr.x on smithi138 2024-07-28T19:38:30.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: cluster 2024-07-28T19:38:29.359959+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-07-28T19:38:30.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: cluster 2024-07-28T19:38:29.604768+0000 mgr.y (mgr.25358) 132 : cluster [DBG] pgmap v227: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 39 KiB/s wr, 468 op/s 2024-07-28T19:38:30.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: audit 2024-07-28T19:38:30.158348+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:30.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: audit 2024-07-28T19:38:30.168130+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:30.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: cephadm 2024-07-28T19:38:30.170016+0000 mgr.y (mgr.25358) 133 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-07-28T19:38:30.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: audit 2024-07-28T19:38:30.170724+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-07-28T19:38:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: audit 2024-07-28T19:38:30.172585+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-07-28T19:38:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: audit 2024-07-28T19:38:30.174034+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:30 smithi138 bash[67894]: cephadm 2024-07-28T19:38:30.175574+0000 mgr.y (mgr.25358) 134 : cephadm [INF] Reconfiguring daemon mgr.x on smithi138 2024-07-28T19:38:30.813 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (353 ms) 2024-07-28T19:38:30.813 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-07-28T19:38:31.183 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (369 ms) 2024-07-28T19:38:31.183 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-07-28T19:38:31.549 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (367 ms) 2024-07-28T19:38:31.549 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-07-28T19:38:31.893 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (343 ms) 2024-07-28T19:38:31.893 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-07-28T19:38:32.352 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (460 ms) 2024-07-28T19:38:32.353 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-07-28T19:38:32.530 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (178 ms) 2024-07-28T19:38:32.530 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-07-28T19:38:32.656 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:32 smithi138 bash[60615]: ts=2024-07-28T19:38:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:32.711 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (181 ms) 2024-07-28T19:38:32.711 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-07-28T19:38:32.870 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (159 ms) 2024-07-28T19:38:32.871 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[82644]: cluster 2024-07-28T19:38:31.605515+0000 mgr.y (mgr.25358) 135 : cluster [DBG] pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 258 KiB/s rd, 31 KiB/s wr, 374 op/s 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[82644]: audit 2024-07-28T19:38:32.400146+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[82644]: audit 2024-07-28T19:38:32.414152+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[82644]: audit 2024-07-28T19:38:32.416391+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[82644]: audit 2024-07-28T19:38:32.418069+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[80199]: cluster 2024-07-28T19:38:31.605515+0000 mgr.y (mgr.25358) 135 : cluster [DBG] pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 258 KiB/s rd, 31 KiB/s wr, 374 op/s 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[80199]: audit 2024-07-28T19:38:32.400146+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[80199]: audit 2024-07-28T19:38:32.414152+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:32.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[80199]: audit 2024-07-28T19:38:32.416391+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:38:32.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[80199]: audit 2024-07-28T19:38:32.418069+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:32.996 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:38:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:38:32] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.43.0" 2024-07-28T19:38:33.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:32 smithi138 bash[67894]: cluster 2024-07-28T19:38:31.605515+0000 mgr.y (mgr.25358) 135 : cluster [DBG] pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 258 KiB/s rd, 31 KiB/s wr, 374 op/s 2024-07-28T19:38:33.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:32 smithi138 bash[67894]: audit 2024-07-28T19:38:32.400146+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:33.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:32 smithi138 bash[67894]: audit 2024-07-28T19:38:32.414152+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:33.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:32 smithi138 bash[67894]: audit 2024-07-28T19:38:32.416391+0000 mon.a (mon.0) 411 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:38:33.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:32 smithi138 bash[67894]: audit 2024-07-28T19:38:32.418069+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:33.113 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (242 ms) 2024-07-28T19:38:33.113 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-07-28T19:38:33.383 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (269 ms) 2024-07-28T19:38:33.383 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-07-28T19:38:33.636 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (253 ms) 2024-07-28T19:38:33.636 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-07-28T19:38:33.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:33 smithi138 bash[67894]: cephadm 2024-07-28T19:38:32.415948+0000 mgr.y (mgr.25358) 136 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-07-28T19:38:33.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:33 smithi138 bash[67894]: cephadm 2024-07-28T19:38:32.421563+0000 mgr.y (mgr.25358) 137 : cephadm [INF] Reconfiguring daemon osd.4 on smithi138 2024-07-28T19:38:33.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:33 smithi138 bash[67894]: audit 2024-07-28T19:38:32.685125+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:33.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:33 smithi138 bash[60615]: ts=2024-07-28T19:38:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:33.880 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (244 ms) 2024-07-28T19:38:33.880 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-07-28T19:38:33.994 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:33 smithi100 bash[80199]: cephadm 2024-07-28T19:38:32.415948+0000 mgr.y (mgr.25358) 136 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-07-28T19:38:33.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:33 smithi100 bash[80199]: cephadm 2024-07-28T19:38:32.421563+0000 mgr.y (mgr.25358) 137 : cephadm [INF] Reconfiguring daemon osd.4 on smithi138 2024-07-28T19:38:33.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:33 smithi100 bash[80199]: audit 2024-07-28T19:38:32.685125+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:33.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:33 smithi100 bash[82644]: cephadm 2024-07-28T19:38:32.415948+0000 mgr.y (mgr.25358) 136 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-07-28T19:38:33.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:33 smithi100 bash[82644]: cephadm 2024-07-28T19:38:32.421563+0000 mgr.y (mgr.25358) 137 : cephadm [INF] Reconfiguring daemon osd.4 on smithi138 2024-07-28T19:38:33.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:33 smithi100 bash[82644]: audit 2024-07-28T19:38:32.685125+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:34.016 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.MirrorStatusList (136 ms) 2024-07-28T19:38:34.016 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-07-28T19:38:34.225 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.RemoveBootstrapped (210 ms) 2024-07-28T19:38:34.225 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-07-28T19:38:34.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:34 smithi138 bash[67894]: cluster 2024-07-28T19:38:33.606976+0000 mgr.y (mgr.25358) 138 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 610 KiB/s rd, 75 KiB/s wr, 859 op/s 2024-07-28T19:38:34.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:34 smithi138 bash[60615]: ts=2024-07-28T19:38:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:34.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:34 smithi138 bash[60615]: ts=2024-07-28T19:38:34.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:34.900 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.AioPromoteDemote (675 ms) 2024-07-28T19:38:34.900 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-07-28T19:38:34.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:34 smithi100 bash[80199]: cluster 2024-07-28T19:38:33.606976+0000 mgr.y (mgr.25358) 138 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 610 KiB/s rd, 75 KiB/s wr, 859 op/s 2024-07-28T19:38:34.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:34 smithi100 bash[82644]: cluster 2024-07-28T19:38:33.606976+0000 mgr.y (mgr.25358) 138 : cluster [DBG] pgmap v229: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 610 KiB/s rd, 75 KiB/s wr, 859 op/s 2024-07-28T19:38:35.520 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.AioGetInfo (620 ms) 2024-07-28T19:38:35.520 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-07-28T19:38:35.924 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.AioGetStatus (404 ms) 2024-07-28T19:38:35.925 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.SiteName 2024-07-28T19:38:35.934 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.SiteName (10 ms) 2024-07-28T19:38:35.934 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.Bootstrap 2024-07-28T19:38:36.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:35 smithi138 bash[67894]: audit 2024-07-28T19:38:34.797876+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:36.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:35 smithi138 bash[67894]: audit 2024-07-28T19:38:34.802956+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:36.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:35 smithi138 bash[67894]: cephadm 2024-07-28T19:38:34.803959+0000 mgr.y (mgr.25358) 139 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-07-28T19:38:36.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:35 smithi138 bash[67894]: audit 2024-07-28T19:38:34.804285+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-28T19:38:36.055 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:35 smithi138 bash[67894]: audit 2024-07-28T19:38:34.805074+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:36.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:35 smithi138 bash[67894]: cephadm 2024-07-28T19:38:34.808371+0000 mgr.y (mgr.25358) 140 : cephadm [INF] Reconfiguring daemon osd.5 on smithi138 2024-07-28T19:38:36.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:35 smithi138 bash[67894]: cluster 2024-07-28T19:38:34.811662+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[80199]: audit 2024-07-28T19:38:34.797876+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[80199]: audit 2024-07-28T19:38:34.802956+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[80199]: cephadm 2024-07-28T19:38:34.803959+0000 mgr.y (mgr.25358) 139 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[80199]: audit 2024-07-28T19:38:34.804285+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[80199]: audit 2024-07-28T19:38:34.805074+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[80199]: cephadm 2024-07-28T19:38:34.808371+0000 mgr.y (mgr.25358) 140 : cephadm [INF] Reconfiguring daemon osd.5 on smithi138 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[80199]: cluster 2024-07-28T19:38:34.811662+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[82644]: audit 2024-07-28T19:38:34.797876+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:36.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[82644]: audit 2024-07-28T19:38:34.802956+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:36.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[82644]: cephadm 2024-07-28T19:38:34.803959+0000 mgr.y (mgr.25358) 139 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-07-28T19:38:36.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[82644]: audit 2024-07-28T19:38:34.804285+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-07-28T19:38:36.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[82644]: audit 2024-07-28T19:38:34.805074+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:36.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[82644]: cephadm 2024-07-28T19:38:34.808371+0000 mgr.y (mgr.25358) 140 : cephadm [INF] Reconfiguring daemon osd.5 on smithi138 2024-07-28T19:38:36.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:35 smithi100 bash[82644]: cluster 2024-07-28T19:38:34.811662+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-07-28T19:38:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[82644]: cluster 2024-07-28T19:38:35.607520+0000 mgr.y (mgr.25358) 141 : cluster [DBG] pgmap v231: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 648 KiB/s rd, 79 KiB/s wr, 914 op/s 2024-07-28T19:38:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[82644]: audit 2024-07-28T19:38:35.934823+0000 mon.a (mon.0) 419 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' 2024-07-28T19:38:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[82644]: audit 2024-07-28T19:38:35.935301+0000 mon.a (mon.0) 420 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[82644]: audit 2024-07-28T19:38:35.935604+0000 mon.a (mon.0) 421 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[82644]: audit 2024-07-28T19:38:35.939498+0000 mon.a (mon.0) 422 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-07-28T19:38:37.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[82644]: audit 2024-07-28T19:38:35.939975+0000 mon.a (mon.0) 423 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[80199]: cluster 2024-07-28T19:38:35.607520+0000 mgr.y (mgr.25358) 141 : cluster [DBG] pgmap v231: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 648 KiB/s rd, 79 KiB/s wr, 914 op/s 2024-07-28T19:38:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[80199]: audit 2024-07-28T19:38:35.934823+0000 mon.a (mon.0) 419 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' 2024-07-28T19:38:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[80199]: audit 2024-07-28T19:38:35.935301+0000 mon.a (mon.0) 420 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[80199]: audit 2024-07-28T19:38:35.935604+0000 mon.a (mon.0) 421 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[80199]: audit 2024-07-28T19:38:35.939498+0000 mon.a (mon.0) 422 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-07-28T19:38:37.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:36 smithi100 bash[80199]: audit 2024-07-28T19:38:35.939975+0000 mon.a (mon.0) 423 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:36 smithi138 bash[67894]: cluster 2024-07-28T19:38:35.607520+0000 mgr.y (mgr.25358) 141 : cluster [DBG] pgmap v231: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 648 KiB/s rd, 79 KiB/s wr, 914 op/s 2024-07-28T19:38:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:36 smithi138 bash[67894]: audit 2024-07-28T19:38:35.934823+0000 mon.a (mon.0) 419 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' 2024-07-28T19:38:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:36 smithi138 bash[67894]: audit 2024-07-28T19:38:35.935301+0000 mon.a (mon.0) 420 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:36 smithi138 bash[67894]: audit 2024-07-28T19:38:35.935604+0000 mon.a (mon.0) 421 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:36 smithi138 bash[67894]: audit 2024-07-28T19:38:35.939498+0000 mon.a (mon.0) 422 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-07-28T19:38:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:36 smithi138 bash[67894]: audit 2024-07-28T19:38:35.939975+0000 mon.a (mon.0) 423 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:37.308 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.Bootstrap (1373 ms) 2024-07-28T19:38:37.308 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.PeerDirection 2024-07-28T19:38:37.319 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.PeerDirection (11 ms) 2024-07-28T19:38:37.320 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.Snapshot 2024-07-28T19:38:38.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.285268+0000 mon.a (mon.0) 424 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-07-28T19:38:38.268 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.289659+0000 mon.a (mon.0) 425 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' 2024-07-28T19:38:38.268 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.290151+0000 mon.a (mon.0) 426 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.294471+0000 mon.a (mon.0) 427 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.294970+0000 mon.a (mon.0) 428 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.307911+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.313397+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: cephadm 2024-07-28T19:38:37.314513+0000 mgr.y (mgr.25358) 142 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.314889+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.315902+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: cephadm 2024-07-28T19:38:37.318473+0000 mgr.y (mgr.25358) 143 : cephadm [INF] Reconfiguring daemon osd.6 on smithi138 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.319333+0000 mon.a (mon.0) 433 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d2943669-a97a-40d1-a64e-62bb0f385266"}]: dispatch 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.322650+0000 mon.a (mon.0) 434 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d2943669-a97a-40d1-a64e-62bb0f385266"}]': finished 2024-07-28T19:38:38.269 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[80199]: audit 2024-07-28T19:38:37.629048+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.285268+0000 mon.a (mon.0) 424 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.289659+0000 mon.a (mon.0) 425 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.290151+0000 mon.a (mon.0) 426 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.294471+0000 mon.a (mon.0) 427 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.294970+0000 mon.a (mon.0) 428 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.307911+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.313397+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: cephadm 2024-07-28T19:38:37.314513+0000 mgr.y (mgr.25358) 142 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.314889+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.315902+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: cephadm 2024-07-28T19:38:37.318473+0000 mgr.y (mgr.25358) 143 : cephadm [INF] Reconfiguring daemon osd.6 on smithi138 2024-07-28T19:38:38.270 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.319333+0000 mon.a (mon.0) 433 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d2943669-a97a-40d1-a64e-62bb0f385266"}]: dispatch 2024-07-28T19:38:38.271 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.322650+0000 mon.a (mon.0) 434 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d2943669-a97a-40d1-a64e-62bb0f385266"}]': finished 2024-07-28T19:38:38.271 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:37 smithi100 bash[82644]: audit 2024-07-28T19:38:37.629048+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:38.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.285268+0000 mon.a (mon.0) 424 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.289659+0000 mon.a (mon.0) 425 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.290151+0000 mon.a (mon.0) 426 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.294471+0000 mon.a (mon.0) 427 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.294970+0000 mon.a (mon.0) 428 : audit [DBG] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.307911+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.313397+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: cephadm 2024-07-28T19:38:37.314513+0000 mgr.y (mgr.25358) 142 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.314889+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.315902+0000 mon.a (mon.0) 432 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: cephadm 2024-07-28T19:38:37.318473+0000 mgr.y (mgr.25358) 143 : cephadm [INF] Reconfiguring daemon osd.6 on smithi138 2024-07-28T19:38:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.319333+0000 mon.a (mon.0) 433 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d2943669-a97a-40d1-a64e-62bb0f385266"}]: dispatch 2024-07-28T19:38:38.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.322650+0000 mon.a (mon.0) 434 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d2943669-a97a-40d1-a64e-62bb0f385266"}]': finished 2024-07-28T19:38:38.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:37 smithi138 bash[67894]: audit 2024-07-28T19:38:37.629048+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:38.910 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:38 smithi138 bash[60615]: ts=2024-07-28T19:38:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:39.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:39 smithi100 bash[80199]: cluster 2024-07-28T19:38:37.608108+0000 mgr.y (mgr.25358) 144 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-07-28T19:38:39.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:39 smithi100 bash[80199]: cluster 2024-07-28T19:38:38.334051+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-07-28T19:38:39.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:39 smithi100 bash[82644]: cluster 2024-07-28T19:38:37.608108+0000 mgr.y (mgr.25358) 144 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-07-28T19:38:39.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:39 smithi100 bash[82644]: cluster 2024-07-28T19:38:38.334051+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-07-28T19:38:39.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:39 smithi138 bash[67894]: cluster 2024-07-28T19:38:37.608108+0000 mgr.y (mgr.25358) 144 : cluster [DBG] pgmap v232: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-07-28T19:38:39.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:39 smithi138 bash[67894]: cluster 2024-07-28T19:38:38.334051+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-07-28T19:38:40.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[80199]: cluster 2024-07-28T19:38:39.343663+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-07-28T19:38:40.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[80199]: audit 2024-07-28T19:38:39.688970+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:40.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[80199]: audit 2024-07-28T19:38:39.700672+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:40.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[80199]: audit 2024-07-28T19:38:39.703319+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-28T19:38:40.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[80199]: audit 2024-07-28T19:38:39.705000+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:40.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[82644]: cluster 2024-07-28T19:38:39.343663+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-07-28T19:38:40.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[82644]: audit 2024-07-28T19:38:39.688970+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:40.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[82644]: audit 2024-07-28T19:38:39.700672+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:40.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[82644]: audit 2024-07-28T19:38:39.703319+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-28T19:38:40.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:40 smithi100 bash[82644]: audit 2024-07-28T19:38:39.705000+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:40 smithi138 bash[67894]: cluster 2024-07-28T19:38:39.343663+0000 mon.a (mon.0) 437 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-07-28T19:38:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:40 smithi138 bash[67894]: audit 2024-07-28T19:38:39.688970+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:40 smithi138 bash[67894]: audit 2024-07-28T19:38:39.700672+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:40.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:40 smithi138 bash[67894]: audit 2024-07-28T19:38:39.703319+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-07-28T19:38:40.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:40 smithi138 bash[67894]: audit 2024-07-28T19:38:39.705000+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:41.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:41 smithi100 bash[80199]: cluster 2024-07-28T19:38:39.608893+0000 mgr.y (mgr.25358) 145 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 857 KiB/s rd, 141 KiB/s wr, 1.16k op/s 2024-07-28T19:38:41.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:41 smithi100 bash[80199]: cephadm 2024-07-28T19:38:39.702727+0000 mgr.y (mgr.25358) 146 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-07-28T19:38:41.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:41 smithi100 bash[80199]: cephadm 2024-07-28T19:38:39.710001+0000 mgr.y (mgr.25358) 147 : cephadm [INF] Reconfiguring daemon osd.7 on smithi138 2024-07-28T19:38:41.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:41 smithi100 bash[80199]: cluster 2024-07-28T19:38:40.357403+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-07-28T19:38:41.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:41 smithi100 bash[82644]: cluster 2024-07-28T19:38:39.608893+0000 mgr.y (mgr.25358) 145 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 857 KiB/s rd, 141 KiB/s wr, 1.16k op/s 2024-07-28T19:38:41.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:41 smithi100 bash[82644]: cephadm 2024-07-28T19:38:39.702727+0000 mgr.y (mgr.25358) 146 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-07-28T19:38:41.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:41 smithi100 bash[82644]: cephadm 2024-07-28T19:38:39.710001+0000 mgr.y (mgr.25358) 147 : cephadm [INF] Reconfiguring daemon osd.7 on smithi138 2024-07-28T19:38:41.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:41 smithi100 bash[82644]: cluster 2024-07-28T19:38:40.357403+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-07-28T19:38:41.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:41 smithi138 bash[67894]: cluster 2024-07-28T19:38:39.608893+0000 mgr.y (mgr.25358) 145 : cluster [DBG] pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 857 KiB/s rd, 141 KiB/s wr, 1.16k op/s 2024-07-28T19:38:41.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:41 smithi138 bash[67894]: cephadm 2024-07-28T19:38:39.702727+0000 mgr.y (mgr.25358) 146 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-07-28T19:38:41.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:41 smithi138 bash[67894]: cephadm 2024-07-28T19:38:39.710001+0000 mgr.y (mgr.25358) 147 : cephadm [INF] Reconfiguring daemon osd.7 on smithi138 2024-07-28T19:38:41.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:41 smithi138 bash[67894]: cluster 2024-07-28T19:38:40.357403+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-07-28T19:38:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[80199]: cluster 2024-07-28T19:38:41.367241+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-07-28T19:38:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[80199]: cluster 2024-07-28T19:38:41.609581+0000 mgr.y (mgr.25358) 148 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 211 KiB/s wr, 1.74k op/s 2024-07-28T19:38:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[80199]: audit 2024-07-28T19:38:41.958160+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:42.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[80199]: audit 2024-07-28T19:38:41.970939+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[80199]: cephadm 2024-07-28T19:38:41.974374+0000 mgr.y (mgr.25358) 149 : cephadm [INF] Reconfiguring mds.foo.smithi138.nphkis (monmap changed)... 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[80199]: audit 2024-07-28T19:38:41.975109+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[80199]: audit 2024-07-28T19:38:41.977287+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[80199]: cephadm 2024-07-28T19:38:41.979285+0000 mgr.y (mgr.25358) 150 : cephadm [INF] Reconfiguring daemon mds.foo.smithi138.nphkis on smithi138 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[82644]: cluster 2024-07-28T19:38:41.367241+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[82644]: cluster 2024-07-28T19:38:41.609581+0000 mgr.y (mgr.25358) 148 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 211 KiB/s wr, 1.74k op/s 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[82644]: audit 2024-07-28T19:38:41.958160+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[82644]: audit 2024-07-28T19:38:41.970939+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:42.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[82644]: cephadm 2024-07-28T19:38:41.974374+0000 mgr.y (mgr.25358) 149 : cephadm [INF] Reconfiguring mds.foo.smithi138.nphkis (monmap changed)... 2024-07-28T19:38:42.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[82644]: audit 2024-07-28T19:38:41.975109+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:38:42.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[82644]: audit 2024-07-28T19:38:41.977287+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:42.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[82644]: cephadm 2024-07-28T19:38:41.979285+0000 mgr.y (mgr.25358) 150 : cephadm [INF] Reconfiguring daemon mds.foo.smithi138.nphkis on smithi138 2024-07-28T19:38:42.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[60615]: ts=2024-07-28T19:38:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:42.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[67894]: cluster 2024-07-28T19:38:41.367241+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-07-28T19:38:42.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[67894]: cluster 2024-07-28T19:38:41.609581+0000 mgr.y (mgr.25358) 148 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 211 KiB/s wr, 1.74k op/s 2024-07-28T19:38:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[67894]: audit 2024-07-28T19:38:41.958160+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[67894]: audit 2024-07-28T19:38:41.970939+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[67894]: cephadm 2024-07-28T19:38:41.974374+0000 mgr.y (mgr.25358) 149 : cephadm [INF] Reconfiguring mds.foo.smithi138.nphkis (monmap changed)... 2024-07-28T19:38:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[67894]: audit 2024-07-28T19:38:41.975109+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.nphkis", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-07-28T19:38:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[67894]: audit 2024-07-28T19:38:41.977287+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:42.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:42 smithi138 bash[67894]: cephadm 2024-07-28T19:38:41.979285+0000 mgr.y (mgr.25358) 150 : cephadm [INF] Reconfiguring daemon mds.foo.smithi138.nphkis on smithi138 2024-07-28T19:38:43.245 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:38:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:38:42] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2024-07-28T19:38:43.617 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:43 smithi138 bash[67894]: cluster 2024-07-28T19:38:42.380457+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-07-28T19:38:43.617 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:43 smithi138 bash[60615]: ts=2024-07-28T19:38:43.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:43.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:43 smithi100 bash[80199]: cluster 2024-07-28T19:38:42.380457+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-07-28T19:38:43.745 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:43 smithi100 bash[82644]: cluster 2024-07-28T19:38:42.380457+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-07-28T19:38:44.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[80199]: cluster 2024-07-28T19:38:43.385088+0000 mon.a (mon.0) 449 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-07-28T19:38:44.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[80199]: cluster 2024-07-28T19:38:43.610965+0000 mgr.y (mgr.25358) 151 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-07-28T19:38:44.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[80199]: audit 2024-07-28T19:38:44.166311+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:44.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[80199]: audit 2024-07-28T19:38:44.178896+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:44.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[80199]: audit 2024-07-28T19:38:44.267644+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:38:44.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[82644]: cluster 2024-07-28T19:38:43.385088+0000 mon.a (mon.0) 449 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-07-28T19:38:44.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[82644]: cluster 2024-07-28T19:38:43.610965+0000 mgr.y (mgr.25358) 151 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-07-28T19:38:44.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[82644]: audit 2024-07-28T19:38:44.166311+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:44.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[82644]: audit 2024-07-28T19:38:44.178896+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:44.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:44 smithi100 bash[82644]: audit 2024-07-28T19:38:44.267644+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:38:44.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:44 smithi138 bash[60615]: ts=2024-07-28T19:38:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:44.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:44 smithi138 bash[60615]: ts=2024-07-28T19:38:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:44.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:44 smithi138 bash[67894]: cluster 2024-07-28T19:38:43.385088+0000 mon.a (mon.0) 449 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-07-28T19:38:44.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:44 smithi138 bash[67894]: cluster 2024-07-28T19:38:43.610965+0000 mgr.y (mgr.25358) 151 : cluster [DBG] pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-07-28T19:38:44.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:44 smithi138 bash[67894]: audit 2024-07-28T19:38:44.166311+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:44.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:44 smithi138 bash[67894]: audit 2024-07-28T19:38:44.178896+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:44.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:44 smithi138 bash[67894]: audit 2024-07-28T19:38:44.267644+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:38:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:45 smithi100 bash[80199]: cluster 2024-07-28T19:38:44.397986+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-07-28T19:38:45.745 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:45 smithi100 bash[80199]: cluster 2024-07-28T19:38:44.816394+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-07-28T19:38:45.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:45 smithi100 bash[82644]: cluster 2024-07-28T19:38:44.397986+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-07-28T19:38:45.746 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:45 smithi100 bash[82644]: cluster 2024-07-28T19:38:44.816394+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-07-28T19:38:45.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:45 smithi138 bash[67894]: cluster 2024-07-28T19:38:44.397986+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-07-28T19:38:45.805 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:45 smithi138 bash[67894]: cluster 2024-07-28T19:38:44.816394+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-07-28T19:38:47.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:46 smithi100 bash[80199]: cluster 2024-07-28T19:38:45.611655+0000 mgr.y (mgr.25358) 152 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-07-28T19:38:47.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:46 smithi100 bash[80199]: cluster 2024-07-28T19:38:45.820696+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-07-28T19:38:47.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:46 smithi100 bash[82644]: cluster 2024-07-28T19:38:45.611655+0000 mgr.y (mgr.25358) 152 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-07-28T19:38:47.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:46 smithi100 bash[82644]: cluster 2024-07-28T19:38:45.820696+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-07-28T19:38:47.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:46 smithi138 bash[67894]: cluster 2024-07-28T19:38:45.611655+0000 mgr.y (mgr.25358) 152 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-07-28T19:38:47.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:46 smithi138 bash[67894]: cluster 2024-07-28T19:38:45.820696+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-07-28T19:38:48.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:47 smithi100 bash[80199]: cluster 2024-07-28T19:38:46.825441+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-07-28T19:38:48.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:47 smithi100 bash[82644]: cluster 2024-07-28T19:38:46.825441+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-07-28T19:38:48.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:47 smithi138 bash[67894]: cluster 2024-07-28T19:38:46.825441+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-07-28T19:38:49.055 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:48 smithi138 bash[60615]: ts=2024-07-28T19:38:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", cluster_addr=\"172.21.15.100\", device_class=\"ssd\", hostname=\"smithi100\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.100\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:49.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:48 smithi138 bash[67894]: cluster 2024-07-28T19:38:47.612415+0000 mgr.y (mgr.25358) 153 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:38:49.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:48 smithi138 bash[67894]: cluster 2024-07-28T19:38:47.846547+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-07-28T19:38:49.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:48 smithi100 bash[80199]: cluster 2024-07-28T19:38:47.612415+0000 mgr.y (mgr.25358) 153 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:38:49.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:48 smithi100 bash[80199]: cluster 2024-07-28T19:38:47.846547+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-07-28T19:38:49.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:48 smithi100 bash[82644]: cluster 2024-07-28T19:38:47.612415+0000 mgr.y (mgr.25358) 153 : cluster [DBG] pgmap v247: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:38:49.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:48 smithi100 bash[82644]: cluster 2024-07-28T19:38:47.846547+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-07-28T19:38:50.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:49 smithi100 bash[80199]: cluster 2024-07-28T19:38:48.852701+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-07-28T19:38:50.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:49 smithi100 bash[80199]: audit 2024-07-28T19:38:49.834958+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:50.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:49 smithi100 bash[80199]: audit 2024-07-28T19:38:49.840575+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:50.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:49 smithi100 bash[82644]: cluster 2024-07-28T19:38:48.852701+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-07-28T19:38:50.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:49 smithi100 bash[82644]: audit 2024-07-28T19:38:49.834958+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:50.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:49 smithi100 bash[82644]: audit 2024-07-28T19:38:49.840575+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:50.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:49 smithi138 bash[67894]: cluster 2024-07-28T19:38:48.852701+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-07-28T19:38:50.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:49 smithi138 bash[67894]: audit 2024-07-28T19:38:49.834958+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:50.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:49 smithi138 bash[67894]: audit 2024-07-28T19:38:49.840575+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[82644]: cluster 2024-07-28T19:38:49.612992+0000 mgr.y (mgr.25358) 154 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-07-28T19:38:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[82644]: cluster 2024-07-28T19:38:49.857180+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-07-28T19:38:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[82644]: audit 2024-07-28T19:38:50.282599+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[82644]: audit 2024-07-28T19:38:50.284263+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:38:51.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[82644]: audit 2024-07-28T19:38:50.294219+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:51.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[80199]: cluster 2024-07-28T19:38:49.612992+0000 mgr.y (mgr.25358) 154 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-07-28T19:38:51.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[80199]: cluster 2024-07-28T19:38:49.857180+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-07-28T19:38:51.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[80199]: audit 2024-07-28T19:38:50.282599+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:51.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[80199]: audit 2024-07-28T19:38:50.284263+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:38:51.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:50 smithi100 bash[80199]: audit 2024-07-28T19:38:50.294219+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:50 smithi138 bash[67894]: cluster 2024-07-28T19:38:49.612992+0000 mgr.y (mgr.25358) 154 : cluster [DBG] pgmap v250: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-07-28T19:38:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:50 smithi138 bash[67894]: cluster 2024-07-28T19:38:49.857180+0000 mon.a (mon.0) 461 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-07-28T19:38:51.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:50 smithi138 bash[67894]: audit 2024-07-28T19:38:50.282599+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:38:51.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:50 smithi138 bash[67894]: audit 2024-07-28T19:38:50.284263+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:38:51.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:50 smithi138 bash[67894]: audit 2024-07-28T19:38:50.294219+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:38:52.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:51 smithi100 bash[80199]: cluster 2024-07-28T19:38:50.869190+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-07-28T19:38:52.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:51 smithi100 bash[82644]: cluster 2024-07-28T19:38:50.869190+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-07-28T19:38:52.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:51 smithi138 bash[67894]: cluster 2024-07-28T19:38:50.869190+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-07-28T19:38:52.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:52 smithi138 bash[60615]: ts=2024-07-28T19:38:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:53.092 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.Snapshot (15773 ms) 2024-07-28T19:38:53.092 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-07-28T19:38:53.245 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:38:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:38:52] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2024-07-28T19:38:53.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:52 smithi100 bash[80199]: cluster 2024-07-28T19:38:51.613632+0000 mgr.y (mgr.25358) 155 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-07-28T19:38:53.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:52 smithi100 bash[80199]: cluster 2024-07-28T19:38:51.879663+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-07-28T19:38:53.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:52 smithi100 bash[80199]: audit 2024-07-28T19:38:52.629387+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:53.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:52 smithi100 bash[82644]: cluster 2024-07-28T19:38:51.613632+0000 mgr.y (mgr.25358) 155 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-07-28T19:38:53.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:52 smithi100 bash[82644]: cluster 2024-07-28T19:38:51.879663+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-07-28T19:38:53.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:52 smithi100 bash[82644]: audit 2024-07-28T19:38:52.629387+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:52 smithi138 bash[67894]: cluster 2024-07-28T19:38:51.613632+0000 mgr.y (mgr.25358) 155 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-07-28T19:38:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:52 smithi138 bash[67894]: cluster 2024-07-28T19:38:51.879663+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-07-28T19:38:53.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:52 smithi138 bash[67894]: audit 2024-07-28T19:38:52.629387+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:38:53.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:53 smithi138 bash[60615]: ts=2024-07-28T19:38:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi100\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", domainname=\"(none)\", instance=\"smithi100\", job=\"node\", machine=\"x86_64\", nodename=\"smithi100\", release=\"5.15.0-117-generic\", sysname=\"Linux\", version=\"#127-Ubuntu SMP Fri Jul 5 20:13:28 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:54.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:53 smithi100 bash[80199]: cluster 2024-07-28T19:38:52.890298+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-07-28T19:38:54.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:53 smithi100 bash[80199]: audit 2024-07-28T19:38:53.081124+0000 mon.a (mon.0) 469 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ccfa5bbf-8148-4251-8965-70457a54d33a"}]: dispatch 2024-07-28T19:38:54.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:53 smithi100 bash[80199]: audit 2024-07-28T19:38:53.087321+0000 mon.a (mon.0) 470 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ccfa5bbf-8148-4251-8965-70457a54d33a"}]': finished 2024-07-28T19:38:54.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:53 smithi100 bash[82644]: cluster 2024-07-28T19:38:52.890298+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-07-28T19:38:54.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:53 smithi100 bash[82644]: audit 2024-07-28T19:38:53.081124+0000 mon.a (mon.0) 469 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ccfa5bbf-8148-4251-8965-70457a54d33a"}]: dispatch 2024-07-28T19:38:54.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:53 smithi100 bash[82644]: audit 2024-07-28T19:38:53.087321+0000 mon.a (mon.0) 470 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ccfa5bbf-8148-4251-8965-70457a54d33a"}]': finished 2024-07-28T19:38:54.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:53 smithi138 bash[67894]: cluster 2024-07-28T19:38:52.890298+0000 mon.a (mon.0) 468 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-07-28T19:38:54.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:53 smithi138 bash[67894]: audit 2024-07-28T19:38:53.081124+0000 mon.a (mon.0) 469 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ccfa5bbf-8148-4251-8965-70457a54d33a"}]: dispatch 2024-07-28T19:38:54.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:53 smithi138 bash[67894]: audit 2024-07-28T19:38:53.087321+0000 mon.a (mon.0) 470 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/ccfa5bbf-8148-4251-8965-70457a54d33a"}]': finished 2024-07-28T19:38:54.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:54 smithi138 bash[60615]: ts=2024-07-28T19:38:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:54.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:38:54 smithi138 bash[60615]: ts=2024-07-28T19:38:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:38:55.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:55 smithi100 bash[82644]: cluster 2024-07-28T19:38:53.614931+0000 mgr.y (mgr.25358) 156 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.2 KiB/s wr, 112 op/s 2024-07-28T19:38:55.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:55 smithi100 bash[82644]: cluster 2024-07-28T19:38:54.100430+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-07-28T19:38:55.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:55 smithi100 bash[82644]: cluster 2024-07-28T19:38:54.821454+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-07-28T19:38:55.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:55 smithi100 bash[80199]: cluster 2024-07-28T19:38:53.614931+0000 mgr.y (mgr.25358) 156 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.2 KiB/s wr, 112 op/s 2024-07-28T19:38:55.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:55 smithi100 bash[80199]: cluster 2024-07-28T19:38:54.100430+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-07-28T19:38:55.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:55 smithi100 bash[80199]: cluster 2024-07-28T19:38:54.821454+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-07-28T19:38:55.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:55 smithi138 bash[67894]: cluster 2024-07-28T19:38:53.614931+0000 mgr.y (mgr.25358) 156 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.2 KiB/s wr, 112 op/s 2024-07-28T19:38:55.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:55 smithi138 bash[67894]: cluster 2024-07-28T19:38:54.100430+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-07-28T19:38:55.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:55 smithi138 bash[67894]: cluster 2024-07-28T19:38:54.821454+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-07-28T19:38:57.039 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (3947 ms) 2024-07-28T19:38:57.040 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-07-28T19:38:57.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:56 smithi100 bash[80199]: cluster 2024-07-28T19:38:55.615651+0000 mgr.y (mgr.25358) 157 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.2 KiB/s wr, 112 op/s 2024-07-28T19:38:57.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:56 smithi100 bash[80199]: cluster 2024-07-28T19:38:55.822984+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-07-28T19:38:57.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:56 smithi100 bash[82644]: cluster 2024-07-28T19:38:55.615651+0000 mgr.y (mgr.25358) 157 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.2 KiB/s wr, 112 op/s 2024-07-28T19:38:57.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:56 smithi100 bash[82644]: cluster 2024-07-28T19:38:55.822984+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-07-28T19:38:57.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:56 smithi138 bash[67894]: cluster 2024-07-28T19:38:55.615651+0000 mgr.y (mgr.25358) 157 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.2 KiB/s wr, 112 op/s 2024-07-28T19:38:57.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:56 smithi138 bash[67894]: cluster 2024-07-28T19:38:55.822984+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-07-28T19:38:58.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:57 smithi100 bash[80199]: cluster 2024-07-28T19:38:56.834649+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-07-28T19:38:58.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:57 smithi100 bash[80199]: audit 2024-07-28T19:38:57.027188+0000 mon.a (mon.0) 475 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/cf9f2448-9b62-430b-8ee1-f7e62deeb547"}]: dispatch 2024-07-28T19:38:58.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:57 smithi100 bash[80199]: audit 2024-07-28T19:38:57.033540+0000 mon.a (mon.0) 476 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/cf9f2448-9b62-430b-8ee1-f7e62deeb547"}]': finished 2024-07-28T19:38:58.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:57 smithi100 bash[82644]: cluster 2024-07-28T19:38:56.834649+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-07-28T19:38:58.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:57 smithi100 bash[82644]: audit 2024-07-28T19:38:57.027188+0000 mon.a (mon.0) 475 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/cf9f2448-9b62-430b-8ee1-f7e62deeb547"}]: dispatch 2024-07-28T19:38:58.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:57 smithi100 bash[82644]: audit 2024-07-28T19:38:57.033540+0000 mon.a (mon.0) 476 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/cf9f2448-9b62-430b-8ee1-f7e62deeb547"}]': finished 2024-07-28T19:38:58.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:57 smithi138 bash[67894]: cluster 2024-07-28T19:38:56.834649+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-07-28T19:38:58.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:57 smithi138 bash[67894]: audit 2024-07-28T19:38:57.027188+0000 mon.a (mon.0) 475 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/cf9f2448-9b62-430b-8ee1-f7e62deeb547"}]: dispatch 2024-07-28T19:38:58.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:57 smithi138 bash[67894]: audit 2024-07-28T19:38:57.033540+0000 mon.a (mon.0) 476 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/cf9f2448-9b62-430b-8ee1-f7e62deeb547"}]': finished 2024-07-28T19:38:59.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:59 smithi138 bash[67894]: cluster 2024-07-28T19:38:57.616324+0000 mgr.y (mgr.25358) 158 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:38:59.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:38:59 smithi138 bash[67894]: cluster 2024-07-28T19:38:58.041814+0000 mon.a (mon.0) 477 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-07-28T19:38:59.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:59 smithi100 bash[80199]: cluster 2024-07-28T19:38:57.616324+0000 mgr.y (mgr.25358) 158 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:38:59.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:38:59 smithi100 bash[80199]: cluster 2024-07-28T19:38:58.041814+0000 mon.a (mon.0) 477 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-07-28T19:38:59.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:59 smithi100 bash[82644]: cluster 2024-07-28T19:38:57.616324+0000 mgr.y (mgr.25358) 158 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:38:59.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:38:59 smithi100 bash[82644]: cluster 2024-07-28T19:38:58.041814+0000 mon.a (mon.0) 477 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-07-28T19:39:00.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:00 smithi138 bash[67894]: cluster 2024-07-28T19:38:59.055242+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-07-28T19:39:00.405 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:00 smithi100 bash[82644]: cluster 2024-07-28T19:38:59.055242+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-07-28T19:39:00.405 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:00 smithi100 bash[80199]: cluster 2024-07-28T19:38:59.055242+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-07-28T19:39:01.113 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch upgrade resume' 2024-07-28T19:39:01.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:01 smithi100 bash[82644]: cluster 2024-07-28T19:38:59.617143+0000 mgr.y (mgr.25358) 159 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 116 op/s 2024-07-28T19:39:01.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:01 smithi100 bash[82644]: cluster 2024-07-28T19:39:00.070281+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-07-28T19:39:01.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:01 smithi100 bash[80199]: cluster 2024-07-28T19:38:59.617143+0000 mgr.y (mgr.25358) 159 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 116 op/s 2024-07-28T19:39:01.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:01 smithi100 bash[80199]: cluster 2024-07-28T19:39:00.070281+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-07-28T19:39:01.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:01 smithi138 bash[67894]: cluster 2024-07-28T19:38:59.617143+0000 mgr.y (mgr.25358) 159 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 116 op/s 2024-07-28T19:39:01.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:01 smithi138 bash[67894]: cluster 2024-07-28T19:39:00.070281+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-07-28T19:39:02.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:02 smithi100 bash[80199]: cluster 2024-07-28T19:39:01.071021+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-07-28T19:39:02.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:02 smithi100 bash[82644]: cluster 2024-07-28T19:39:01.071021+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-07-28T19:39:02.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:02 smithi138 bash[67894]: cluster 2024-07-28T19:39:01.071021+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-07-28T19:39:02.555 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:39:02 smithi138 bash[60615]: ts=2024-07-28T19:39:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:39:03.245 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:39:02] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2024-07-28T19:39:03.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:03 smithi100 bash[82644]: cluster 2024-07-28T19:39:01.617798+0000 mgr.y (mgr.25358) 160 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 116 op/s 2024-07-28T19:39:03.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:03 smithi100 bash[82644]: cluster 2024-07-28T19:39:02.091417+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-07-28T19:39:03.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:03 smithi100 bash[82644]: audit 2024-07-28T19:39:02.127991+0000 mon.a (mon.0) 482 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/14e385db-4343-416b-9fe2-0c9c9e766d76"}]: dispatch 2024-07-28T19:39:03.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:03 smithi100 bash[82644]: audit 2024-07-28T19:39:02.134817+0000 mon.a (mon.0) 483 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/14e385db-4343-416b-9fe2-0c9c9e766d76"}]': finished 2024-07-28T19:39:03.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:03 smithi100 bash[80199]: cluster 2024-07-28T19:39:01.617798+0000 mgr.y (mgr.25358) 160 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 116 op/s 2024-07-28T19:39:03.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:03 smithi100 bash[80199]: cluster 2024-07-28T19:39:02.091417+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-07-28T19:39:03.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:03 smithi100 bash[80199]: audit 2024-07-28T19:39:02.127991+0000 mon.a (mon.0) 482 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/14e385db-4343-416b-9fe2-0c9c9e766d76"}]: dispatch 2024-07-28T19:39:03.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:03 smithi100 bash[80199]: audit 2024-07-28T19:39:02.134817+0000 mon.a (mon.0) 483 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/14e385db-4343-416b-9fe2-0c9c9e766d76"}]': finished 2024-07-28T19:39:03.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:03 smithi138 bash[67894]: cluster 2024-07-28T19:39:01.617798+0000 mgr.y (mgr.25358) 160 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 116 op/s 2024-07-28T19:39:03.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:03 smithi138 bash[67894]: cluster 2024-07-28T19:39:02.091417+0000 mon.a (mon.0) 481 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-07-28T19:39:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:03 smithi138 bash[67894]: audit 2024-07-28T19:39:02.127991+0000 mon.a (mon.0) 482 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/14e385db-4343-416b-9fe2-0c9c9e766d76"}]: dispatch 2024-07-28T19:39:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:03 smithi138 bash[67894]: audit 2024-07-28T19:39:02.134817+0000 mon.a (mon.0) 483 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/14e385db-4343-416b-9fe2-0c9c9e766d76"}]': finished 2024-07-28T19:39:04.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:04 smithi100 bash[80199]: cluster 2024-07-28T19:39:03.140147+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-07-28T19:39:04.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:04 smithi100 bash[82644]: cluster 2024-07-28T19:39:03.140147+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-07-28T19:39:04.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:04 smithi138 bash[67894]: cluster 2024-07-28T19:39:03.140147+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-07-28T19:39:05.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:05 smithi100 bash[80199]: cluster 2024-07-28T19:39:03.619173+0000 mgr.y (mgr.25358) 161 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 183 KiB/s rd, 11 KiB/s wr, 239 op/s 2024-07-28T19:39:05.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:05 smithi100 bash[80199]: cluster 2024-07-28T19:39:04.153689+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-07-28T19:39:05.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:05 smithi100 bash[80199]: cluster 2024-07-28T19:39:04.824571+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-07-28T19:39:05.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:05 smithi100 bash[82644]: cluster 2024-07-28T19:39:03.619173+0000 mgr.y (mgr.25358) 161 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 183 KiB/s rd, 11 KiB/s wr, 239 op/s 2024-07-28T19:39:05.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:05 smithi100 bash[82644]: cluster 2024-07-28T19:39:04.153689+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-07-28T19:39:05.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:05 smithi100 bash[82644]: cluster 2024-07-28T19:39:04.824571+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-07-28T19:39:05.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:05 smithi138 bash[67894]: cluster 2024-07-28T19:39:03.619173+0000 mgr.y (mgr.25358) 161 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 183 KiB/s rd, 11 KiB/s wr, 239 op/s 2024-07-28T19:39:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:05 smithi138 bash[67894]: cluster 2024-07-28T19:39:04.153689+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-07-28T19:39:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:05 smithi138 bash[67894]: cluster 2024-07-28T19:39:04.824571+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-07-28T19:39:05.899 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:39:06.986 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (9946 ms) 2024-07-28T19:39:06.986 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-07-28T19:39:07.198 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:06 smithi100 bash[80199]: cluster 2024-07-28T19:39:05.619928+0000 mgr.y (mgr.25358) 162 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 183 KiB/s rd, 11 KiB/s wr, 239 op/s 2024-07-28T19:39:07.198 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:06 smithi100 bash[80199]: cluster 2024-07-28T19:39:05.829282+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-07-28T19:39:07.198 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:06 smithi100 bash[82644]: cluster 2024-07-28T19:39:05.619928+0000 mgr.y (mgr.25358) 162 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 183 KiB/s rd, 11 KiB/s wr, 239 op/s 2024-07-28T19:39:07.198 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:06 smithi100 bash[82644]: cluster 2024-07-28T19:39:05.829282+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-07-28T19:39:07.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:06 smithi138 bash[67894]: cluster 2024-07-28T19:39:05.619928+0000 mgr.y (mgr.25358) 162 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 183 KiB/s rd, 11 KiB/s wr, 239 op/s 2024-07-28T19:39:07.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:06 smithi138 bash[67894]: cluster 2024-07-28T19:39:05.829282+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-07-28T19:39:08.147 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[80199]: cluster 2024-07-28T19:39:06.836262+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-07-28T19:39:08.147 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[80199]: audit 2024-07-28T19:39:06.968034+0000 mon.a (mon.0) 489 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4d738b29-a59b-4464-b3df-b6a0432f30c1"}]: dispatch 2024-07-28T19:39:08.147 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[80199]: audit 2024-07-28T19:39:06.973988+0000 mon.a (mon.0) 490 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4d738b29-a59b-4464-b3df-b6a0432f30c1"}]': finished 2024-07-28T19:39:08.147 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[80199]: audit 2024-07-28T19:39:06.978617+0000 mon.a (mon.0) 491 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/8816ad7c-30dd-4ec1-9bf9-c9fbf4125c0f"}]: dispatch 2024-07-28T19:39:08.148 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[80199]: audit 2024-07-28T19:39:06.983438+0000 mon.a (mon.0) 492 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/8816ad7c-30dd-4ec1-9bf9-c9fbf4125c0f"}]': finished 2024-07-28T19:39:08.148 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[80199]: audit 2024-07-28T19:39:07.629854+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:08.148 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[82644]: cluster 2024-07-28T19:39:06.836262+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-07-28T19:39:08.148 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[82644]: audit 2024-07-28T19:39:06.968034+0000 mon.a (mon.0) 489 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4d738b29-a59b-4464-b3df-b6a0432f30c1"}]: dispatch 2024-07-28T19:39:08.148 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[82644]: audit 2024-07-28T19:39:06.973988+0000 mon.a (mon.0) 490 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4d738b29-a59b-4464-b3df-b6a0432f30c1"}]': finished 2024-07-28T19:39:08.148 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[82644]: audit 2024-07-28T19:39:06.978617+0000 mon.a (mon.0) 491 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/8816ad7c-30dd-4ec1-9bf9-c9fbf4125c0f"}]: dispatch 2024-07-28T19:39:08.148 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[82644]: audit 2024-07-28T19:39:06.983438+0000 mon.a (mon.0) 492 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/8816ad7c-30dd-4ec1-9bf9-c9fbf4125c0f"}]': finished 2024-07-28T19:39:08.148 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:07 smithi100 bash[82644]: audit 2024-07-28T19:39:07.629854+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:08.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:07 smithi138 bash[67894]: cluster 2024-07-28T19:39:06.836262+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-07-28T19:39:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:07 smithi138 bash[67894]: audit 2024-07-28T19:39:06.968034+0000 mon.a (mon.0) 489 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4d738b29-a59b-4464-b3df-b6a0432f30c1"}]: dispatch 2024-07-28T19:39:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:07 smithi138 bash[67894]: audit 2024-07-28T19:39:06.973988+0000 mon.a (mon.0) 490 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/4d738b29-a59b-4464-b3df-b6a0432f30c1"}]': finished 2024-07-28T19:39:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:07 smithi138 bash[67894]: audit 2024-07-28T19:39:06.978617+0000 mon.a (mon.0) 491 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/8816ad7c-30dd-4ec1-9bf9-c9fbf4125c0f"}]: dispatch 2024-07-28T19:39:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:07 smithi138 bash[67894]: audit 2024-07-28T19:39:06.983438+0000 mon.a (mon.0) 492 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/8816ad7c-30dd-4ec1-9bf9-c9fbf4125c0f"}]': finished 2024-07-28T19:39:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:07 smithi138 bash[67894]: audit 2024-07-28T19:39:07.629854+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:09.256 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:09 smithi100 bash[80199]: cluster 2024-07-28T19:39:07.620599+0000 mgr.y (mgr.25358) 163 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:09.257 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:09 smithi100 bash[80199]: cluster 2024-07-28T19:39:08.002530+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-07-28T19:39:09.257 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:09 smithi100 bash[82644]: cluster 2024-07-28T19:39:07.620599+0000 mgr.y (mgr.25358) 163 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:09.257 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:09 smithi100 bash[82644]: cluster 2024-07-28T19:39:08.002530+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-07-28T19:39:09.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:09 smithi138 bash[67894]: cluster 2024-07-28T19:39:07.620599+0000 mgr.y (mgr.25358) 163 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:09.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:09 smithi138 bash[67894]: cluster 2024-07-28T19:39:08.002530+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-07-28T19:39:09.597 INFO:teuthology.orchestra.run.smithi100.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:39:10.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:10 smithi138 bash[67894]: cluster 2024-07-28T19:39:09.002989+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-07-28T19:39:10.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:10 smithi138 bash[67894]: audit 2024-07-28T19:39:09.602053+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:10 smithi138 bash[67894]: audit 2024-07-28T19:39:09.603033+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:10.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:10 smithi100 bash[80199]: cluster 2024-07-28T19:39:09.002989+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-07-28T19:39:10.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:10 smithi100 bash[80199]: audit 2024-07-28T19:39:09.602053+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:10.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:10 smithi100 bash[80199]: audit 2024-07-28T19:39:09.603033+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:10.346 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:10 smithi100 bash[82644]: cluster 2024-07-28T19:39:09.002989+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-07-28T19:39:10.346 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:10 smithi100 bash[82644]: audit 2024-07-28T19:39:09.602053+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:10.346 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:10 smithi100 bash[82644]: audit 2024-07-28T19:39:09.603033+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:10.347 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'echo wait for all mons to upgrade' 2024-07-28T19:39:11.131 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.SnapshotImageState (4145 ms) 2024-07-28T19:39:11.131 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-07-28T19:39:11.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:11 smithi138 bash[67894]: audit 2024-07-28T19:39:09.597307+0000 mgr.y (mgr.25358) 164 : audit [DBG] from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:11 smithi138 bash[67894]: cephadm 2024-07-28T19:39:09.597731+0000 mgr.y (mgr.25358) 165 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:39:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:11 smithi138 bash[67894]: cluster 2024-07-28T19:39:09.621002+0000 mgr.y (mgr.25358) 166 : cluster [DBG] pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-07-28T19:39:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:11 smithi138 bash[67894]: cluster 2024-07-28T19:39:10.024840+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-07-28T19:39:11.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:11 smithi100 bash[80199]: audit 2024-07-28T19:39:09.597307+0000 mgr.y (mgr.25358) 164 : audit [DBG] from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:11.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:11 smithi100 bash[80199]: cephadm 2024-07-28T19:39:09.597731+0000 mgr.y (mgr.25358) 165 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:39:11.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:11 smithi100 bash[80199]: cluster 2024-07-28T19:39:09.621002+0000 mgr.y (mgr.25358) 166 : cluster [DBG] pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-07-28T19:39:11.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:11 smithi100 bash[80199]: cluster 2024-07-28T19:39:10.024840+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-07-28T19:39:11.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:11 smithi100 bash[82644]: audit 2024-07-28T19:39:09.597307+0000 mgr.y (mgr.25358) 164 : audit [DBG] from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:11.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:11 smithi100 bash[82644]: cephadm 2024-07-28T19:39:09.597731+0000 mgr.y (mgr.25358) 165 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:39:11.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:11 smithi100 bash[82644]: cluster 2024-07-28T19:39:09.621002+0000 mgr.y (mgr.25358) 166 : cluster [DBG] pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-07-28T19:39:11.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:11 smithi100 bash[82644]: cluster 2024-07-28T19:39:10.024840+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-07-28T19:39:12.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:12 smithi138 bash[67894]: cluster 2024-07-28T19:39:11.036278+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-07-28T19:39:12.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:12 smithi100 bash[80199]: cluster 2024-07-28T19:39:11.036278+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-07-28T19:39:12.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:12 smithi100 bash[82644]: cluster 2024-07-28T19:39:11.036278+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-07-28T19:39:12.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:39:12 smithi138 bash[60615]: ts=2024-07-28T19:39:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:39:13.245 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:39:12] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2024-07-28T19:39:13.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:13 smithi100 bash[80199]: cluster 2024-07-28T19:39:11.621621+0000 mgr.y (mgr.25358) 167 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-07-28T19:39:13.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:13 smithi100 bash[80199]: cluster 2024-07-28T19:39:12.049255+0000 mon.a (mon.0) 500 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-07-28T19:39:13.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:13 smithi100 bash[82644]: cluster 2024-07-28T19:39:11.621621+0000 mgr.y (mgr.25358) 167 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-07-28T19:39:13.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:13 smithi100 bash[82644]: cluster 2024-07-28T19:39:12.049255+0000 mon.a (mon.0) 500 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-07-28T19:39:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:13 smithi138 bash[67894]: cluster 2024-07-28T19:39:11.621621+0000 mgr.y (mgr.25358) 167 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s rd, 7.7 KiB/s wr, 108 op/s 2024-07-28T19:39:13.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:13 smithi138 bash[67894]: cluster 2024-07-28T19:39:12.049255+0000 mon.a (mon.0) 500 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-07-28T19:39:14.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:14 smithi100 bash[80199]: cluster 2024-07-28T19:39:13.052977+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-07-28T19:39:14.495 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:14 smithi100 bash[82644]: cluster 2024-07-28T19:39:13.052977+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-07-28T19:39:14.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:14 smithi138 bash[67894]: cluster 2024-07-28T19:39:13.052977+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-07-28T19:39:15.134 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:39:15.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:15 smithi100 bash[80199]: cluster 2024-07-28T19:39:13.622946+0000 mgr.y (mgr.25358) 168 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 25 KiB/s wr, 251 op/s 2024-07-28T19:39:15.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:15 smithi100 bash[80199]: cluster 2024-07-28T19:39:14.076487+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-07-28T19:39:15.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:15 smithi100 bash[80199]: cluster 2024-07-28T19:39:14.826718+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-07-28T19:39:15.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:15 smithi100 bash[82644]: cluster 2024-07-28T19:39:13.622946+0000 mgr.y (mgr.25358) 168 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 25 KiB/s wr, 251 op/s 2024-07-28T19:39:15.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:15 smithi100 bash[82644]: cluster 2024-07-28T19:39:14.076487+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-07-28T19:39:15.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:15 smithi100 bash[82644]: cluster 2024-07-28T19:39:14.826718+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-07-28T19:39:15.555 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:15 smithi138 bash[67894]: cluster 2024-07-28T19:39:13.622946+0000 mgr.y (mgr.25358) 168 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 25 KiB/s wr, 251 op/s 2024-07-28T19:39:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:15 smithi138 bash[67894]: cluster 2024-07-28T19:39:14.076487+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-07-28T19:39:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:15 smithi138 bash[67894]: cluster 2024-07-28T19:39:14.826718+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-07-28T19:39:16.728 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:15.473730+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.728 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:15.484567+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.728 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: cluster 2024-07-28T19:39:15.623652+0000 mgr.y (mgr.25358) 169 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 25 KiB/s wr, 251 op/s 2024-07-28T19:39:16.728 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: cluster 2024-07-28T19:39:15.829684+0000 mon.a (mon.0) 506 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-07-28T19:39:16.728 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:15.989667+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:16.728 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:15.991453+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:39:16.728 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:15.999966+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.093940+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.098165+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.100655+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: cephadm 2024-07-28T19:39:16.101662+0000 mgr.y (mgr.25358) 170 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.113411+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.115239+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.121171+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.122869+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.131721+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.133388+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.140822+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-07-28T19:39:16.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.142831+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:15.473730+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:15.484567+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: cluster 2024-07-28T19:39:15.623652+0000 mgr.y (mgr.25358) 169 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 25 KiB/s wr, 251 op/s 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: cluster 2024-07-28T19:39:15.829684+0000 mon.a (mon.0) 506 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:15.989667+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:15.991453+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:15.999966+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.093940+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:16.730 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.098165+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.100655+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: cephadm 2024-07-28T19:39:16.101662+0000 mgr.y (mgr.25358) 170 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.113411+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.115239+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.121171+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.122869+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.131721+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.133388+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.140822+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.142831+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: cephadm 2024-07-28T19:39:16.144280+0000 mgr.y (mgr.25358) 171 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-07-28T19:39:16.731 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.153612+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.732 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.157295+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-28T19:39:16.732 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: audit 2024-07-28T19:39:16.157677+0000 mgr.y (mgr.25358) 172 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-28T19:39:16.732 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[80199]: cephadm 2024-07-28T19:39:16.159845+0000 mgr.y (mgr.25358) 173 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-07-28T19:39:16.732 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: cephadm 2024-07-28T19:39:16.144280+0000 mgr.y (mgr.25358) 171 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-07-28T19:39:16.732 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.153612+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.732 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.157295+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-28T19:39:16.732 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: audit 2024-07-28T19:39:16.157677+0000 mgr.y (mgr.25358) 172 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-28T19:39:16.732 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:16 smithi100 bash[82644]: cephadm 2024-07-28T19:39:16.159845+0000 mgr.y (mgr.25358) 173 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-07-28T19:39:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:15.473730+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:15.484567+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: cluster 2024-07-28T19:39:15.623652+0000 mgr.y (mgr.25358) 169 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 178 KiB/s rd, 25 KiB/s wr, 251 op/s 2024-07-28T19:39:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: cluster 2024-07-28T19:39:15.829684+0000 mon.a (mon.0) 506 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:15.989667+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:15.991453+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:15.999966+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.093940+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.098165+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.100655+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: cephadm 2024-07-28T19:39:16.101662+0000 mgr.y (mgr.25358) 170 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.113411+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.115239+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.121171+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.122869+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.131721+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-07-28T19:39:16.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.133388+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-07-28T19:39:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.140822+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-07-28T19:39:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.142831+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: cephadm 2024-07-28T19:39:16.144280+0000 mgr.y (mgr.25358) 171 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-07-28T19:39:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.153612+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.157295+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-28T19:39:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: audit 2024-07-28T19:39:16.157677+0000 mgr.y (mgr.25358) 172 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-07-28T19:39:16.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:16 smithi138 bash[67894]: cephadm 2024-07-28T19:39:16.159845+0000 mgr.y (mgr.25358) 173 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-07-28T19:39:18.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:17 smithi100 bash[80199]: cluster 2024-07-28T19:39:16.829179+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-07-28T19:39:18.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:17 smithi100 bash[82644]: cluster 2024-07-28T19:39:16.829179+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-07-28T19:39:18.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:17 smithi138 bash[67894]: cluster 2024-07-28T19:39:16.829179+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-07-28T19:39:19.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:18 smithi100 bash[80199]: cluster 2024-07-28T19:39:17.624324+0000 mgr.y (mgr.25358) 174 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:19.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:18 smithi100 bash[80199]: cluster 2024-07-28T19:39:17.845761+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-07-28T19:39:19.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:18 smithi100 bash[82644]: cluster 2024-07-28T19:39:17.624324+0000 mgr.y (mgr.25358) 174 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:19.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:18 smithi100 bash[82644]: cluster 2024-07-28T19:39:17.845761+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-07-28T19:39:19.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:18 smithi138 bash[67894]: cluster 2024-07-28T19:39:17.624324+0000 mgr.y (mgr.25358) 174 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:19.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:18 smithi138 bash[67894]: cluster 2024-07-28T19:39:17.845761+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-07-28T19:39:20.107 INFO:teuthology.orchestra.run.smithi100.stdout:wait for all mons to upgrade 2024-07-28T19:39:20.108 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:19 smithi100 bash[80199]: cluster 2024-07-28T19:39:18.848035+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-07-28T19:39:20.108 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:19 smithi100 bash[82644]: cluster 2024-07-28T19:39:18.848035+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-07-28T19:39:20.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:19 smithi138 bash[67894]: cluster 2024-07-28T19:39:18.848035+0000 mon.a (mon.0) 525 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-07-28T19:39:20.990 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (9859 ms) 2024-07-28T19:39:20.990 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[80199]: cluster 2024-07-28T19:39:19.625052+0000 mgr.y (mgr.25358) 175 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 7.7 KiB/s wr, 126 op/s 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[80199]: cluster 2024-07-28T19:39:19.861853+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[80199]: cephadm 2024-07-28T19:39:19.905954+0000 mgr.y (mgr.25358) 176 : cephadm [INF] Upgrade: Updating osd.0 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[80199]: audit 2024-07-28T19:39:19.915055+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[80199]: audit 2024-07-28T19:39:19.916459+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[80199]: audit 2024-07-28T19:39:19.917553+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[80199]: cephadm 2024-07-28T19:39:19.920934+0000 mgr.y (mgr.25358) 177 : cephadm [INF] Deploying daemon osd.0 on smithi100 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[82644]: cluster 2024-07-28T19:39:19.625052+0000 mgr.y (mgr.25358) 175 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 7.7 KiB/s wr, 126 op/s 2024-07-28T19:39:21.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[82644]: cluster 2024-07-28T19:39:19.861853+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-07-28T19:39:21.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[82644]: cephadm 2024-07-28T19:39:19.905954+0000 mgr.y (mgr.25358) 176 : cephadm [INF] Upgrade: Updating osd.0 2024-07-28T19:39:21.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[82644]: audit 2024-07-28T19:39:19.915055+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:21.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[82644]: audit 2024-07-28T19:39:19.916459+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:39:21.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[82644]: audit 2024-07-28T19:39:19.917553+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:21.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:20 smithi100 bash[82644]: cephadm 2024-07-28T19:39:19.920934+0000 mgr.y (mgr.25358) 177 : cephadm [INF] Deploying daemon osd.0 on smithi100 2024-07-28T19:39:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:20 smithi138 bash[67894]: cluster 2024-07-28T19:39:19.625052+0000 mgr.y (mgr.25358) 175 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 7.7 KiB/s wr, 126 op/s 2024-07-28T19:39:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:20 smithi138 bash[67894]: cluster 2024-07-28T19:39:19.861853+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2024-07-28T19:39:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:20 smithi138 bash[67894]: cephadm 2024-07-28T19:39:19.905954+0000 mgr.y (mgr.25358) 176 : cephadm [INF] Upgrade: Updating osd.0 2024-07-28T19:39:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:20 smithi138 bash[67894]: audit 2024-07-28T19:39:19.915055+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:20 smithi138 bash[67894]: audit 2024-07-28T19:39:19.916459+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-07-28T19:39:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:20 smithi138 bash[67894]: audit 2024-07-28T19:39:19.917553+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:20 smithi138 bash[67894]: cephadm 2024-07-28T19:39:19.920934+0000 mgr.y (mgr.25358) 177 : cephadm [INF] Deploying daemon osd.0 on smithi100 2024-07-28T19:39:21.817 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-07-28T19:39:22.085 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:21 smithi100 bash[80199]: cluster 2024-07-28T19:39:20.870859+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-07-28T19:39:22.085 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:21 smithi100 bash[80199]: audit 2024-07-28T19:39:20.977280+0000 mon.a (mon.0) 531 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e2f1419f-9085-4e1a-8b8b-3d13c7541929"}]: dispatch 2024-07-28T19:39:22.085 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:21 smithi100 bash[80199]: audit 2024-07-28T19:39:20.983734+0000 mon.a (mon.0) 532 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e2f1419f-9085-4e1a-8b8b-3d13c7541929"}]': finished 2024-07-28T19:39:22.085 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:21 smithi100 bash[82644]: cluster 2024-07-28T19:39:20.870859+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-07-28T19:39:22.086 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:21 smithi100 bash[82644]: audit 2024-07-28T19:39:20.977280+0000 mon.a (mon.0) 531 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e2f1419f-9085-4e1a-8b8b-3d13c7541929"}]: dispatch 2024-07-28T19:39:22.086 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:21 smithi100 bash[82644]: audit 2024-07-28T19:39:20.983734+0000 mon.a (mon.0) 532 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e2f1419f-9085-4e1a-8b8b-3d13c7541929"}]': finished 2024-07-28T19:39:22.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:21 smithi138 bash[67894]: cluster 2024-07-28T19:39:20.870859+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2024-07-28T19:39:22.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:21 smithi138 bash[67894]: audit 2024-07-28T19:39:20.977280+0000 mon.a (mon.0) 531 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e2f1419f-9085-4e1a-8b8b-3d13c7541929"}]: dispatch 2024-07-28T19:39:22.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:21 smithi138 bash[67894]: audit 2024-07-28T19:39:20.983734+0000 mon.a (mon.0) 532 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e2f1419f-9085-4e1a-8b8b-3d13c7541929"}]': finished 2024-07-28T19:39:22.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:39:22 smithi138 bash[60615]: ts=2024-07-28T19:39:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:39:23.166 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:39:22] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2024-07-28T19:39:23.167 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:22 smithi100 bash[80199]: cluster 2024-07-28T19:39:21.625674+0000 mgr.y (mgr.25358) 178 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 7.7 KiB/s wr, 126 op/s 2024-07-28T19:39:23.167 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:22 smithi100 bash[80199]: cluster 2024-07-28T19:39:21.992124+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-07-28T19:39:23.167 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:22 smithi100 bash[80199]: audit 2024-07-28T19:39:22.630291+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:23.167 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:22 smithi100 bash[82644]: cluster 2024-07-28T19:39:21.625674+0000 mgr.y (mgr.25358) 178 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 7.7 KiB/s wr, 126 op/s 2024-07-28T19:39:23.167 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:22 smithi100 bash[82644]: cluster 2024-07-28T19:39:21.992124+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-07-28T19:39:23.167 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:22 smithi100 bash[82644]: audit 2024-07-28T19:39:22.630291+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:23.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:22 smithi138 bash[67894]: cluster 2024-07-28T19:39:21.625674+0000 mgr.y (mgr.25358) 178 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 7.7 KiB/s wr, 126 op/s 2024-07-28T19:39:23.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:22 smithi138 bash[67894]: cluster 2024-07-28T19:39:21.992124+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2024-07-28T19:39:23.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:22 smithi138 bash[67894]: audit 2024-07-28T19:39:22.630291+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:24.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:23 smithi100 bash[80199]: cluster 2024-07-28T19:39:23.000739+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-07-28T19:39:24.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:24 smithi100 bash[82644]: cluster 2024-07-28T19:39:23.000739+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-07-28T19:39:24.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:23 smithi138 bash[67894]: cluster 2024-07-28T19:39:23.000739+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2024-07-28T19:39:25.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:25 smithi138 bash[67894]: cluster 2024-07-28T19:39:23.626946+0000 mgr.y (mgr.25358) 179 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 10 KiB/s wr, 93 op/s 2024-07-28T19:39:25.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:25 smithi138 bash[67894]: cluster 2024-07-28T19:39:24.013375+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-07-28T19:39:25.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:25 smithi138 bash[67894]: cluster 2024-07-28T19:39:24.836135+0000 mon.a (mon.0) 537 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-07-28T19:39:25.343 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[80199]: cluster 2024-07-28T19:39:23.626946+0000 mgr.y (mgr.25358) 179 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 10 KiB/s wr, 93 op/s 2024-07-28T19:39:25.343 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[80199]: cluster 2024-07-28T19:39:24.013375+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-07-28T19:39:25.343 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[80199]: cluster 2024-07-28T19:39:24.836135+0000 mon.a (mon.0) 537 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-07-28T19:39:25.343 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[82644]: cluster 2024-07-28T19:39:23.626946+0000 mgr.y (mgr.25358) 179 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 10 KiB/s wr, 93 op/s 2024-07-28T19:39:25.344 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[82644]: cluster 2024-07-28T19:39:24.013375+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2024-07-28T19:39:25.344 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[82644]: cluster 2024-07-28T19:39:24.836135+0000 mon.a (mon.0) 537 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2024-07-28T19:39:25.635 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.635 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.635 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.636 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.636 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.636 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.637 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.637 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.637 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: Stopping Ceph osd.0 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:39:25.637 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:39:25 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:25.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[80199]: debug 2024-07-28T19:39:25.821+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:39:25.996 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[34708]: debug 2024-07-28T19:39:25.761+0000 7fb3b491b640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:39:25.996 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[34708]: debug 2024-07-28T19:39:25.761+0000 7fb3b491b640 -1 osd.0 490 *** Got signal Terminated *** 2024-07-28T19:39:25.996 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:25 smithi100 bash[34708]: debug 2024-07-28T19:39:25.761+0000 7fb3b491b640 -1 osd.0 490 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:39:26.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:26 smithi138 bash[67894]: cluster 2024-07-28T19:39:25.767716+0000 mon.a (mon.0) 538 : cluster [INF] osd.0 marked itself down and dead 2024-07-28T19:39:26.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:26 smithi138 bash[67894]: cluster 2024-07-28T19:39:25.825126+0000 mon.a (mon.0) 539 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:39:26.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:26 smithi138 bash[67894]: cluster 2024-07-28T19:39:25.825171+0000 mon.a (mon.0) 540 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:39:26.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:26 smithi138 bash[67894]: cluster 2024-07-28T19:39:25.830439+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-07-28T19:39:26.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:26 smithi100 bash[80199]: cluster 2024-07-28T19:39:25.767716+0000 mon.a (mon.0) 538 : cluster [INF] osd.0 marked itself down and dead 2024-07-28T19:39:26.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:26 smithi100 bash[80199]: cluster 2024-07-28T19:39:25.825126+0000 mon.a (mon.0) 539 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:39:26.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:26 smithi100 bash[80199]: cluster 2024-07-28T19:39:25.825171+0000 mon.a (mon.0) 540 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:39:26.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:26 smithi100 bash[80199]: cluster 2024-07-28T19:39:25.830439+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-07-28T19:39:26.395 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:26 smithi100 bash[82644]: cluster 2024-07-28T19:39:25.767716+0000 mon.a (mon.0) 538 : cluster [INF] osd.0 marked itself down and dead 2024-07-28T19:39:26.395 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:26 smithi100 bash[82644]: cluster 2024-07-28T19:39:25.825126+0000 mon.a (mon.0) 539 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:39:26.395 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:26 smithi100 bash[82644]: cluster 2024-07-28T19:39:25.825171+0000 mon.a (mon.0) 540 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:39:26.395 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:26 smithi100 bash[82644]: cluster 2024-07-28T19:39:25.830439+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-07-28T19:39:26.608 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:39:27.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:27 smithi138 bash[67894]: cluster 2024-07-28T19:39:25.627623+0000 mgr.y (mgr.25358) 180 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 10 KiB/s wr, 93 op/s 2024-07-28T19:39:27.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:27 smithi138 bash[67894]: cluster 2024-07-28T19:39:26.839310+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e492: 8 total, 7 up, 8 in 2024-07-28T19:39:27.495 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:27 smithi100 bash[89007]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-0 2024-07-28T19:39:27.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:27 smithi100 bash[82644]: cluster 2024-07-28T19:39:25.627623+0000 mgr.y (mgr.25358) 180 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 10 KiB/s wr, 93 op/s 2024-07-28T19:39:27.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:27 smithi100 bash[82644]: cluster 2024-07-28T19:39:26.839310+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e492: 8 total, 7 up, 8 in 2024-07-28T19:39:27.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:27 smithi100 bash[80199]: cluster 2024-07-28T19:39:25.627623+0000 mgr.y (mgr.25358) 180 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 10 KiB/s wr, 93 op/s 2024-07-28T19:39:27.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:27 smithi100 bash[80199]: cluster 2024-07-28T19:39:26.839310+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e492: 8 total, 7 up, 8 in 2024-07-28T19:39:27.920 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:27 smithi100 bash[89086]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-0 2024-07-28T19:39:29.134 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:28 smithi100 bash[82644]: cluster 2024-07-28T19:39:27.628150+0000 mgr.y (mgr.25358) 181 : cluster [DBG] pgmap v307: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:29.134 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:28 smithi100 bash[82644]: cluster 2024-07-28T19:39:27.840433+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e493: 8 total, 7 up, 8 in 2024-07-28T19:39:29.135 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:28 smithi100 bash[80199]: cluster 2024-07-28T19:39:27.628150+0000 mgr.y (mgr.25358) 181 : cluster [DBG] pgmap v307: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:29.135 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:28 smithi100 bash[80199]: cluster 2024-07-28T19:39:27.840433+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e493: 8 total, 7 up, 8 in 2024-07-28T19:39:29.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:28 smithi138 bash[67894]: cluster 2024-07-28T19:39:27.628150+0000 mgr.y (mgr.25358) 181 : cluster [DBG] pgmap v307: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:29.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:28 smithi138 bash[67894]: cluster 2024-07-28T19:39:27.840433+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e493: 8 total, 7 up, 8 in 2024-07-28T19:39:30.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:29 smithi100 bash[82644]: cluster 2024-07-28T19:39:28.849663+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e494: 8 total, 7 up, 8 in 2024-07-28T19:39:30.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:29 smithi100 bash[80199]: cluster 2024-07-28T19:39:28.849663+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e494: 8 total, 7 up, 8 in 2024-07-28T19:39:30.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:29 smithi138 bash[67894]: cluster 2024-07-28T19:39:28.849663+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e494: 8 total, 7 up, 8 in 2024-07-28T19:39:31.135 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.136 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.136 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.136 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.137 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:30 smithi100 bash[80199]: cluster 2024-07-28T19:39:29.628881+0000 mgr.y (mgr.25358) 182 : cluster [DBG] pgmap v310: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 42/396 objects degraded (10.606%) 2024-07-28T19:39:31.137 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:30 smithi100 bash[80199]: cluster 2024-07-28T19:39:29.843741+0000 mon.a (mon.0) 545 : cluster [WRN] Health check failed: Degraded data redundancy: 42/396 objects degraded (10.606%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:31.137 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:30 smithi100 bash[80199]: cluster 2024-07-28T19:39:29.863025+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e495: 8 total, 7 up, 8 in 2024-07-28T19:39:31.137 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.137 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.137 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:30 smithi100 bash[82644]: cluster 2024-07-28T19:39:29.628881+0000 mgr.y (mgr.25358) 182 : cluster [DBG] pgmap v310: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 42/396 objects degraded (10.606%) 2024-07-28T19:39:31.138 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:30 smithi100 bash[82644]: cluster 2024-07-28T19:39:29.843741+0000 mon.a (mon.0) 545 : cluster [WRN] Health check failed: Degraded data redundancy: 42/396 objects degraded (10.606%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:31.138 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:30 smithi100 bash[82644]: cluster 2024-07-28T19:39:29.863025+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e495: 8 total, 7 up, 8 in 2024-07-28T19:39:31.138 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.138 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:30 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.0.service: Deactivated successfully. 2024-07-28T19:39:31.138 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:30 smithi100 systemd[1]: Stopped Ceph osd.0 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:39:31.138 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.138 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: Started Ceph osd.0 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:39:31.139 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:39:31 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:30 smithi138 bash[67894]: cluster 2024-07-28T19:39:29.628881+0000 mgr.y (mgr.25358) 182 : cluster [DBG] pgmap v310: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 42/396 objects degraded (10.606%) 2024-07-28T19:39:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:30 smithi138 bash[67894]: cluster 2024-07-28T19:39:29.843741+0000 mon.a (mon.0) 545 : cluster [WRN] Health check failed: Degraded data redundancy: 42/396 objects degraded (10.606%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:30 smithi138 bash[67894]: cluster 2024-07-28T19:39:29.863025+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e495: 8 total, 7 up, 8 in 2024-07-28T19:39:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:31 smithi100 bash[80199]: cluster 2024-07-28T19:39:30.867080+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e496: 8 total, 7 up, 8 in 2024-07-28T19:39:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:31 smithi100 bash[80199]: audit 2024-07-28T19:39:31.171454+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:31 smithi100 bash[80199]: audit 2024-07-28T19:39:31.177458+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:31 smithi100 bash[80199]: audit 2024-07-28T19:39:31.178516+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:32.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:31 smithi100 bash[82644]: cluster 2024-07-28T19:39:30.867080+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e496: 8 total, 7 up, 8 in 2024-07-28T19:39:32.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:31 smithi100 bash[82644]: audit 2024-07-28T19:39:31.171454+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:32.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:31 smithi100 bash[82644]: audit 2024-07-28T19:39:31.177458+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:32.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:31 smithi100 bash[82644]: audit 2024-07-28T19:39:31.178516+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:32.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:31 smithi138 bash[67894]: cluster 2024-07-28T19:39:30.867080+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e496: 8 total, 7 up, 8 in 2024-07-28T19:39:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:31 smithi138 bash[67894]: audit 2024-07-28T19:39:31.171454+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:31 smithi138 bash[67894]: audit 2024-07-28T19:39:31.177458+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:31 smithi138 bash[67894]: audit 2024-07-28T19:39:31.178516+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:32.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:39:32 smithi138 bash[60615]: ts=2024-07-28T19:39:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:39:32.868 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable)": 3 2024-07-28T19:39:33.128 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:32 smithi100 bash[82644]: cluster 2024-07-28T19:39:31.629733+0000 mgr.y (mgr.25358) 183 : cluster [DBG] pgmap v313: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 42/396 objects degraded (10.606%) 2024-07-28T19:39:33.128 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:32 smithi100 bash[82644]: cluster 2024-07-28T19:39:31.884860+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e497: 8 total, 7 up, 8 in 2024-07-28T19:39:33.128 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:32 smithi100 bash[82644]: audit 2024-07-28T19:39:32.862191+0000 mon.a (mon.0) 552 : audit [DBG] from='client.? 172.21.15.100:0/2379578730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:39:33.128 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:32 smithi100 bash[80199]: cluster 2024-07-28T19:39:31.629733+0000 mgr.y (mgr.25358) 183 : cluster [DBG] pgmap v313: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 42/396 objects degraded (10.606%) 2024-07-28T19:39:33.128 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:32 smithi100 bash[80199]: cluster 2024-07-28T19:39:31.884860+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e497: 8 total, 7 up, 8 in 2024-07-28T19:39:33.128 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:32 smithi100 bash[80199]: audit 2024-07-28T19:39:32.862191+0000 mon.a (mon.0) 552 : audit [DBG] from='client.? 172.21.15.100:0/2379578730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:39:33.128 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:39:32] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2024-07-28T19:39:33.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:32 smithi138 bash[67894]: cluster 2024-07-28T19:39:31.629733+0000 mgr.y (mgr.25358) 183 : cluster [DBG] pgmap v313: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 42/396 objects degraded (10.606%) 2024-07-28T19:39:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:32 smithi138 bash[67894]: cluster 2024-07-28T19:39:31.884860+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e497: 8 total, 7 up, 8 in 2024-07-28T19:39:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:32 smithi138 bash[67894]: audit 2024-07-28T19:39:32.862191+0000 mon.a (mon.0) 552 : audit [DBG] from='client.? 172.21.15.100:0/2379578730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-07-28T19:39:33.607 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch ps' 2024-07-28T19:39:33.892 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[89302]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-07-28T19:39:33.893 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[89302]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-07-28T19:39:33.893 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[89302]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-07-28T19:39:33.893 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[89302]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-07-28T19:39:33.893 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[89302]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-07-28T19:39:33.893 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[89302]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-07-28T19:39:33.893 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[89302]: --> ceph-volume raw activate successful for osd ID: 0 2024-07-28T19:39:34.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[80199]: cluster 2024-07-28T19:39:32.891399+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e498: 8 total, 7 up, 8 in 2024-07-28T19:39:34.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:33 smithi100 bash[82644]: cluster 2024-07-28T19:39:32.891399+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e498: 8 total, 7 up, 8 in 2024-07-28T19:39:34.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:33 smithi138 bash[67894]: cluster 2024-07-28T19:39:32.891399+0000 mon.a (mon.0) 553 : cluster [DBG] osdmap e498: 8 total, 7 up, 8 in 2024-07-28T19:39:35.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:34 smithi100 bash[80199]: cluster 2024-07-28T19:39:33.630798+0000 mgr.y (mgr.25358) 184 : cluster [DBG] pgmap v316: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 24 KiB/s wr, 245 op/s; 58/501 objects degraded (11.577%) 2024-07-28T19:39:35.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:34 smithi100 bash[80199]: cluster 2024-07-28T19:39:33.902222+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e499: 8 total, 7 up, 8 in 2024-07-28T19:39:35.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:34 smithi100 bash[82644]: cluster 2024-07-28T19:39:33.630798+0000 mgr.y (mgr.25358) 184 : cluster [DBG] pgmap v316: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 24 KiB/s wr, 245 op/s; 58/501 objects degraded (11.577%) 2024-07-28T19:39:35.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:34 smithi100 bash[82644]: cluster 2024-07-28T19:39:33.902222+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e499: 8 total, 7 up, 8 in 2024-07-28T19:39:35.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:34 smithi138 bash[67894]: cluster 2024-07-28T19:39:33.630798+0000 mgr.y (mgr.25358) 184 : cluster [DBG] pgmap v316: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 24 KiB/s wr, 245 op/s; 58/501 objects degraded (11.577%) 2024-07-28T19:39:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:34 smithi138 bash[67894]: cluster 2024-07-28T19:39:33.902222+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e499: 8 total, 7 up, 8 in 2024-07-28T19:39:35.745 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:35 smithi100 bash[89627]: debug 2024-07-28T19:39:35.517+0000 7f936fa90640 1 -- 172.21.15.100:0/2831257017 <== mon.0 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55646a3a51e0 con 0x55646b122000 2024-07-28T19:39:36.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:35 smithi100 bash[80199]: cluster 2024-07-28T19:39:34.910986+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e500: 8 total, 7 up, 8 in 2024-07-28T19:39:36.246 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:36 smithi100 bash[89627]: debug 2024-07-28T19:39:36.053+0000 7f93722f6640 -1 Falling back to public interface 2024-07-28T19:39:36.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:35 smithi100 bash[82644]: cluster 2024-07-28T19:39:34.910986+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e500: 8 total, 7 up, 8 in 2024-07-28T19:39:36.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:35 smithi138 bash[67894]: cluster 2024-07-28T19:39:34.910986+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e500: 8 total, 7 up, 8 in 2024-07-28T19:39:37.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:36 smithi100 bash[80199]: cluster 2024-07-28T19:39:35.631683+0000 mgr.y (mgr.25358) 185 : cluster [DBG] pgmap v319: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 24 KiB/s wr, 245 op/s; 58/501 objects degraded (11.577%) 2024-07-28T19:39:37.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:36 smithi100 bash[80199]: cluster 2024-07-28T19:39:35.920347+0000 mon.a (mon.0) 556 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2024-07-28T19:39:37.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:36 smithi100 bash[82644]: cluster 2024-07-28T19:39:35.631683+0000 mgr.y (mgr.25358) 185 : cluster [DBG] pgmap v319: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 24 KiB/s wr, 245 op/s; 58/501 objects degraded (11.577%) 2024-07-28T19:39:37.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:36 smithi100 bash[82644]: cluster 2024-07-28T19:39:35.920347+0000 mon.a (mon.0) 556 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2024-07-28T19:39:37.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:36 smithi138 bash[67894]: cluster 2024-07-28T19:39:35.631683+0000 mgr.y (mgr.25358) 185 : cluster [DBG] pgmap v319: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 24 KiB/s wr, 245 op/s; 58/501 objects degraded (11.577%) 2024-07-28T19:39:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:36 smithi138 bash[67894]: cluster 2024-07-28T19:39:35.920347+0000 mon.a (mon.0) 556 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2024-07-28T19:39:38.245 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[82644]: cluster 2024-07-28T19:39:36.934882+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e502: 8 total, 7 up, 8 in 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[82644]: audit 2024-07-28T19:39:37.633357+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[82644]: audit 2024-07-28T19:39:37.634167+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[82644]: audit 2024-07-28T19:39:37.782384+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[82644]: audit 2024-07-28T19:39:37.788260+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[80199]: cluster 2024-07-28T19:39:36.934882+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e502: 8 total, 7 up, 8 in 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[80199]: audit 2024-07-28T19:39:37.633357+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[80199]: audit 2024-07-28T19:39:37.634167+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[80199]: audit 2024-07-28T19:39:37.782384+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:38.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:37 smithi100 bash[80199]: audit 2024-07-28T19:39:37.788260+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:37 smithi138 bash[67894]: cluster 2024-07-28T19:39:36.934882+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e502: 8 total, 7 up, 8 in 2024-07-28T19:39:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:37 smithi138 bash[67894]: audit 2024-07-28T19:39:37.633357+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:37 smithi138 bash[67894]: audit 2024-07-28T19:39:37.634167+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:37 smithi138 bash[67894]: audit 2024-07-28T19:39:37.782384+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:38.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:37 smithi138 bash[67894]: audit 2024-07-28T19:39:37.788260+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:39.205 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:39:39.245 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:39 smithi100 bash[89627]: debug 2024-07-28T19:39:39.121+0000 7f93722f6640 -1 osd.0 490 log_to_monitors true 2024-07-28T19:39:39.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:38 smithi100 bash[80199]: cluster 2024-07-28T19:39:37.632285+0000 mgr.y (mgr.25358) 186 : cluster [DBG] pgmap v322: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58/501 objects degraded (11.577%) 2024-07-28T19:39:39.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:38 smithi100 bash[80199]: cluster 2024-07-28T19:39:37.947642+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e503: 8 total, 7 up, 8 in 2024-07-28T19:39:39.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:38 smithi100 bash[80199]: audit 2024-07-28T19:39:38.481402+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:39.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:38 smithi100 bash[80199]: audit 2024-07-28T19:39:38.487040+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:39.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:38 smithi100 bash[82644]: cluster 2024-07-28T19:39:37.632285+0000 mgr.y (mgr.25358) 186 : cluster [DBG] pgmap v322: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58/501 objects degraded (11.577%) 2024-07-28T19:39:39.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:38 smithi100 bash[82644]: cluster 2024-07-28T19:39:37.947642+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e503: 8 total, 7 up, 8 in 2024-07-28T19:39:39.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:38 smithi100 bash[82644]: audit 2024-07-28T19:39:38.481402+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:39.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:38 smithi100 bash[82644]: audit 2024-07-28T19:39:38.487040+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:39.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:38 smithi138 bash[67894]: cluster 2024-07-28T19:39:37.632285+0000 mgr.y (mgr.25358) 186 : cluster [DBG] pgmap v322: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 585 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58/501 objects degraded (11.577%) 2024-07-28T19:39:39.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:38 smithi138 bash[67894]: cluster 2024-07-28T19:39:37.947642+0000 mon.a (mon.0) 562 : cluster [DBG] osdmap e503: 8 total, 7 up, 8 in 2024-07-28T19:39:39.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:38 smithi138 bash[67894]: audit 2024-07-28T19:39:38.481402+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:39.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:38 smithi138 bash[67894]: audit 2024-07-28T19:39:38.487040+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:40.245 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:39 smithi100 bash[89627]: debug 2024-07-28T19:39:39.993+0000 7f936a0a0640 -1 osd.0 490 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-28T19:39:40.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:39 smithi100 bash[80199]: cluster 2024-07-28T19:39:38.955804+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e504: 8 total, 7 up, 8 in 2024-07-28T19:39:40.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:39 smithi100 bash[80199]: audit 2024-07-28T19:39:39.129402+0000 mon.a (mon.0) 566 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:39:40.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:39 smithi100 bash[80199]: cluster 2024-07-28T19:39:39.829665+0000 mon.a (mon.0) 567 : cluster [WRN] Health check update: Degraded data redundancy: 58/501 objects degraded (11.577%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:40.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:39 smithi100 bash[82644]: cluster 2024-07-28T19:39:38.955804+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e504: 8 total, 7 up, 8 in 2024-07-28T19:39:40.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:39 smithi100 bash[82644]: audit 2024-07-28T19:39:39.129402+0000 mon.a (mon.0) 566 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:39:40.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:39 smithi100 bash[82644]: cluster 2024-07-28T19:39:39.829665+0000 mon.a (mon.0) 567 : cluster [WRN] Health check update: Degraded data redundancy: 58/501 objects degraded (11.577%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:40.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:39 smithi138 bash[67894]: cluster 2024-07-28T19:39:38.955804+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e504: 8 total, 7 up, 8 in 2024-07-28T19:39:40.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:39 smithi138 bash[67894]: audit 2024-07-28T19:39:39.129402+0000 mon.a (mon.0) 566 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-07-28T19:39:40.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:39 smithi138 bash[67894]: cluster 2024-07-28T19:39:39.829665+0000 mon.a (mon.0) 567 : cluster [WRN] Health check update: Degraded data redundancy: 58/501 objects degraded (11.577%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:40.978 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[80199]: debug 2024-07-28T19:39:40.953+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:39:41.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[80199]: cluster 2024-07-28T19:39:39.633143+0000 mgr.y (mgr.25358) 187 : cluster [DBG] pgmap v325: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s; 55/453 objects degraded (12.141%) 2024-07-28T19:39:41.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[80199]: audit 2024-07-28T19:39:39.963863+0000 mon.a (mon.0) 568 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-28T19:39:41.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[80199]: cluster 2024-07-28T19:39:39.964063+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e505: 8 total, 7 up, 8 in 2024-07-28T19:39:41.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[80199]: audit 2024-07-28T19:39:39.965361+0000 mon.a (mon.0) 570 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:39:41.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[82644]: cluster 2024-07-28T19:39:39.633143+0000 mgr.y (mgr.25358) 187 : cluster [DBG] pgmap v325: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s; 55/453 objects degraded (12.141%) 2024-07-28T19:39:41.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[82644]: audit 2024-07-28T19:39:39.963863+0000 mon.a (mon.0) 568 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-28T19:39:41.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[82644]: cluster 2024-07-28T19:39:39.964063+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e505: 8 total, 7 up, 8 in 2024-07-28T19:39:41.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:40 smithi100 bash[82644]: audit 2024-07-28T19:39:39.965361+0000 mon.a (mon.0) 570 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:39:41.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:40 smithi138 bash[67894]: cluster 2024-07-28T19:39:39.633143+0000 mgr.y (mgr.25358) 187 : cluster [DBG] pgmap v325: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s; 55/453 objects degraded (12.141%) 2024-07-28T19:39:41.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:40 smithi138 bash[67894]: audit 2024-07-28T19:39:39.963863+0000 mon.a (mon.0) 568 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-07-28T19:39:41.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:40 smithi138 bash[67894]: cluster 2024-07-28T19:39:39.964063+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e505: 8 total, 7 up, 8 in 2024-07-28T19:39:41.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:40 smithi138 bash[67894]: audit 2024-07-28T19:39:39.965361+0000 mon.a (mon.0) 570 : audit [INF] from='osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:39:42.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:41 smithi100 bash[80199]: cluster 2024-07-28T19:39:40.958997+0000 mon.a (mon.0) 571 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:39:42.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:41 smithi100 bash[80199]: cluster 2024-07-28T19:39:40.959039+0000 mon.a (mon.0) 572 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:39:42.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:41 smithi100 bash[80199]: cluster 2024-07-28T19:39:40.968636+0000 mon.a (mon.0) 573 : cluster [INF] osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453] boot 2024-07-28T19:39:42.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:41 smithi100 bash[80199]: cluster 2024-07-28T19:39:40.968756+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-07-28T19:39:42.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:41 smithi100 bash[80199]: audit 2024-07-28T19:39:40.971854+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:39:42.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:41 smithi100 bash[80199]: cluster 2024-07-28T19:39:41.971751+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-07-28T19:39:42.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[82644]: cluster 2024-07-28T19:39:40.958997+0000 mon.a (mon.0) 571 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:39:42.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[82644]: cluster 2024-07-28T19:39:40.959039+0000 mon.a (mon.0) 572 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:39:42.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[82644]: cluster 2024-07-28T19:39:40.968636+0000 mon.a (mon.0) 573 : cluster [INF] osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453] boot 2024-07-28T19:39:42.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[82644]: cluster 2024-07-28T19:39:40.968756+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-07-28T19:39:42.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[82644]: audit 2024-07-28T19:39:40.971854+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:39:42.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[82644]: cluster 2024-07-28T19:39:41.971751+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-07-28T19:39:42.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:41 smithi138 bash[67894]: cluster 2024-07-28T19:39:40.958997+0000 mon.a (mon.0) 571 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:39:42.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:41 smithi138 bash[67894]: cluster 2024-07-28T19:39:40.959039+0000 mon.a (mon.0) 572 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:39:42.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:41 smithi138 bash[67894]: cluster 2024-07-28T19:39:40.968636+0000 mon.a (mon.0) 573 : cluster [INF] osd.0 [v2:172.21.15.100:6802/2080394453,v1:172.21.15.100:6803/2080394453] boot 2024-07-28T19:39:42.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:41 smithi138 bash[67894]: cluster 2024-07-28T19:39:40.968756+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-07-28T19:39:42.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:41 smithi138 bash[67894]: audit 2024-07-28T19:39:40.971854+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-07-28T19:39:42.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:41 smithi138 bash[67894]: cluster 2024-07-28T19:39:41.971751+0000 mon.a (mon.0) 576 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-07-28T19:39:42.805 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:39:42 smithi138 bash[60615]: ts=2024-07-28T19:39:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:39:42.875 INFO:teuthology.orchestra.run.smithi100.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-28T19:39:42.875 INFO:teuthology.orchestra.run.smithi100.stdout:alertmanager.a smithi100 *:9093,9094 running (5m) 5s ago 12m 13.7M - 0.25.0 c8568f914cd2 61edf7ffcf9d 2024-07-28T19:39:42.875 INFO:teuthology.orchestra.run.smithi100.stdout:grafana.a smithi138 *:3000 running (5m) 27s ago 12m 75.4M - 9.4.7 954c08fa6188 fe0244c1bc8a 2024-07-28T19:39:42.875 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi100.ppcqpk smithi100 running (10m) 5s ago 10m 14.4M - 17.2.7-1155-g3120d060 5ec745195f7e 24cab9a6a84c 2024-07-28T19:39:42.875 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi138.nphkis smithi138 running (10m) 27s ago 10m 18.1M - 17.2.7-1155-g3120d060 5ec745195f7e cc36fa0642f5 2024-07-28T19:39:42.875 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.x smithi138 *:8443,9283,8765 running (4m) 27s ago 21m 424M - 18.2.2-1765-g1c7dd747 dd66d6cc220c 6027bd7b0eab 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.y smithi100 *:8443,9283,8765 running (5m) 5s ago 23m 481M - 18.2.2-1765-g1c7dd747 dd66d6cc220c d4cdb32af700 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:mon.a smithi100 running (4m) 5s ago 23m 54.9M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 82b65d214966 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:mon.b smithi138 running (119s) 27s ago 21m 38.4M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 7d6318daf479 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:mon.c smithi100 running (2m) 5s ago 21m 42.0M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 7a8870df1475 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.a smithi100 *:9100 running (5m) 5s ago 13m 8280k - 1.5.0 0da6a335fe13 558573030b79 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.b smithi138 *:9100 running (5m) 27s ago 13m 8128k - 1.5.0 0da6a335fe13 3867a7654990 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:osd.0 smithi100 running (8s) 5s ago 20m - 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c a2819bfee948 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:osd.1 smithi100 running (19m) 5s ago 19m 123M 2524M 17.2.7-1155-g3120d060 5ec745195f7e ceb5589accd5 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:osd.2 smithi100 running (18m) 5s ago 18m 137M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 1b4df5053189 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:osd.3 smithi100 running (17m) 5s ago 17m 119M 2524M 17.2.7-1155-g3120d060 5ec745195f7e cf8d59f38b6d 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:osd.4 smithi138 running (16m) 27s ago 16m 115M 2524M 17.2.7-1155-g3120d060 5ec745195f7e feff0c636bab 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:osd.5 smithi138 running (15m) 27s ago 16m 107M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 33bea4a7c9ec 2024-07-28T19:39:42.876 INFO:teuthology.orchestra.run.smithi100.stdout:osd.6 smithi138 running (15m) 27s ago 15m 127M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 11e0eaf60c0b 2024-07-28T19:39:42.877 INFO:teuthology.orchestra.run.smithi100.stdout:osd.7 smithi138 running (14m) 27s ago 14m 145M 2524M 17.2.7-1155-g3120d060 5ec745195f7e fd79c6c44f0f 2024-07-28T19:39:42.877 INFO:teuthology.orchestra.run.smithi100.stdout:prometheus.a smithi138 *:9095 running (4m) 27s ago 13m 57.0M - 2.43.0 a07b618ecd1d bc318a7bc21e 2024-07-28T19:39:43.245 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:39:42] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2024-07-28T19:39:43.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[80199]: cluster 2024-07-28T19:39:41.634018+0000 mgr.y (mgr.25358) 188 : cluster [DBG] pgmap v328: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s; 55/453 objects degraded (12.141%) 2024-07-28T19:39:43.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:42 smithi100 bash[82644]: cluster 2024-07-28T19:39:41.634018+0000 mgr.y (mgr.25358) 188 : cluster [DBG] pgmap v328: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s; 55/453 objects degraded (12.141%) 2024-07-28T19:39:43.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:42 smithi138 bash[67894]: cluster 2024-07-28T19:39:41.634018+0000 mgr.y (mgr.25358) 188 : cluster [DBG] pgmap v328: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s; 55/453 objects degraded (12.141%) 2024-07-28T19:39:43.677 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch upgrade pause' 2024-07-28T19:39:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:44 smithi138 bash[67894]: audit 2024-07-28T19:39:42.842416+0000 mgr.y (mgr.25358) 189 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:44.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:44 smithi138 bash[67894]: cluster 2024-07-28T19:39:43.015219+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-07-28T19:39:44.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:44 smithi100 bash[80199]: audit 2024-07-28T19:39:42.842416+0000 mgr.y (mgr.25358) 189 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:44.395 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:44 smithi100 bash[80199]: cluster 2024-07-28T19:39:43.015219+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-07-28T19:39:44.395 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:44 smithi100 bash[82644]: audit 2024-07-28T19:39:42.842416+0000 mgr.y (mgr.25358) 189 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:44.395 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:44 smithi100 bash[82644]: cluster 2024-07-28T19:39:43.015219+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-07-28T19:39:45.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:45 smithi138 bash[67894]: cluster 2024-07-28T19:39:43.635345+0000 mgr.y (mgr.25358) 190 : cluster [DBG] pgmap v331: 105 pgs: 9 active+undersized, 4 active+undersized+degraded, 92 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 5.5 KiB/s wr, 104 op/s; 14/414 objects degraded (3.382%); 125 B/s, 4 keys/s, 4 objects/s recovering 2024-07-28T19:39:45.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:45 smithi138 bash[67894]: cluster 2024-07-28T19:39:44.023613+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-07-28T19:39:45.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:45 smithi138 bash[67894]: cluster 2024-07-28T19:39:44.830954+0000 mon.a (mon.0) 579 : cluster [WRN] Health check update: Degraded data redundancy: 14/414 objects degraded (3.382%), 4 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:45.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:45 smithi100 bash[82644]: cluster 2024-07-28T19:39:43.635345+0000 mgr.y (mgr.25358) 190 : cluster [DBG] pgmap v331: 105 pgs: 9 active+undersized, 4 active+undersized+degraded, 92 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 5.5 KiB/s wr, 104 op/s; 14/414 objects degraded (3.382%); 125 B/s, 4 keys/s, 4 objects/s recovering 2024-07-28T19:39:45.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:45 smithi100 bash[82644]: cluster 2024-07-28T19:39:44.023613+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-07-28T19:39:45.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:45 smithi100 bash[82644]: cluster 2024-07-28T19:39:44.830954+0000 mon.a (mon.0) 579 : cluster [WRN] Health check update: Degraded data redundancy: 14/414 objects degraded (3.382%), 4 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:45.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:45 smithi100 bash[80199]: cluster 2024-07-28T19:39:43.635345+0000 mgr.y (mgr.25358) 190 : cluster [DBG] pgmap v331: 105 pgs: 9 active+undersized, 4 active+undersized+degraded, 92 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 5.5 KiB/s wr, 104 op/s; 14/414 objects degraded (3.382%); 125 B/s, 4 keys/s, 4 objects/s recovering 2024-07-28T19:39:45.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:45 smithi100 bash[80199]: cluster 2024-07-28T19:39:44.023613+0000 mon.a (mon.0) 578 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-07-28T19:39:45.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:45 smithi100 bash[80199]: cluster 2024-07-28T19:39:44.830954+0000 mon.a (mon.0) 579 : cluster [WRN] Health check update: Degraded data redundancy: 14/414 objects degraded (3.382%), 4 pgs degraded (PG_DEGRADED) 2024-07-28T19:39:46.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:46 smithi100 bash[80199]: cluster 2024-07-28T19:39:45.035088+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-07-28T19:39:46.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:46 smithi100 bash[82644]: cluster 2024-07-28T19:39:45.035088+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-07-28T19:39:46.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:46 smithi138 bash[67894]: cluster 2024-07-28T19:39:45.035088+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-07-28T19:39:47.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:47 smithi138 bash[67894]: cluster 2024-07-28T19:39:45.635834+0000 mgr.y (mgr.25358) 191 : cluster [DBG] pgmap v334: 105 pgs: 105 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.7 KiB/s wr, 137 op/s; 147 B/s, 11 keys/s, 6 objects/s recovering 2024-07-28T19:39:47.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:47 smithi138 bash[67894]: cluster 2024-07-28T19:39:46.028160+0000 mon.a (mon.0) 581 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/414 objects degraded (3.382%), 4 pgs degraded) 2024-07-28T19:39:47.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:47 smithi138 bash[67894]: cluster 2024-07-28T19:39:46.047998+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-07-28T19:39:47.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:47 smithi100 bash[80199]: cluster 2024-07-28T19:39:45.635834+0000 mgr.y (mgr.25358) 191 : cluster [DBG] pgmap v334: 105 pgs: 105 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.7 KiB/s wr, 137 op/s; 147 B/s, 11 keys/s, 6 objects/s recovering 2024-07-28T19:39:47.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:47 smithi100 bash[80199]: cluster 2024-07-28T19:39:46.028160+0000 mon.a (mon.0) 581 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/414 objects degraded (3.382%), 4 pgs degraded) 2024-07-28T19:39:47.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:47 smithi100 bash[80199]: cluster 2024-07-28T19:39:46.047998+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-07-28T19:39:47.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:47 smithi100 bash[82644]: cluster 2024-07-28T19:39:45.635834+0000 mgr.y (mgr.25358) 191 : cluster [DBG] pgmap v334: 105 pgs: 105 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 7.7 KiB/s wr, 137 op/s; 147 B/s, 11 keys/s, 6 objects/s recovering 2024-07-28T19:39:47.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:47 smithi100 bash[82644]: cluster 2024-07-28T19:39:46.028160+0000 mon.a (mon.0) 581 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/414 objects degraded (3.382%), 4 pgs degraded) 2024-07-28T19:39:47.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:47 smithi100 bash[82644]: cluster 2024-07-28T19:39:46.047998+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-07-28T19:39:48.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:48 smithi138 bash[67894]: cluster 2024-07-28T19:39:47.054626+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-07-28T19:39:48.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:48 smithi100 bash[82644]: cluster 2024-07-28T19:39:47.054626+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-07-28T19:39:48.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:48 smithi100 bash[80199]: cluster 2024-07-28T19:39:47.054626+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-07-28T19:39:49.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:49 smithi138 bash[67894]: cluster 2024-07-28T19:39:47.636507+0000 mgr.y (mgr.25358) 192 : cluster [DBG] pgmap v337: 105 pgs: 105 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 KiB/s wr, 32 op/s; 21 B/s, 6 keys/s, 1 objects/s recovering 2024-07-28T19:39:49.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:49 smithi138 bash[67894]: cluster 2024-07-28T19:39:48.061818+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-07-28T19:39:49.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:49 smithi100 bash[80199]: cluster 2024-07-28T19:39:47.636507+0000 mgr.y (mgr.25358) 192 : cluster [DBG] pgmap v337: 105 pgs: 105 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 KiB/s wr, 32 op/s; 21 B/s, 6 keys/s, 1 objects/s recovering 2024-07-28T19:39:49.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:49 smithi100 bash[80199]: cluster 2024-07-28T19:39:48.061818+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-07-28T19:39:49.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:49 smithi100 bash[82644]: cluster 2024-07-28T19:39:47.636507+0000 mgr.y (mgr.25358) 192 : cluster [DBG] pgmap v337: 105 pgs: 105 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 KiB/s wr, 32 op/s; 21 B/s, 6 keys/s, 1 objects/s recovering 2024-07-28T19:39:49.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:49 smithi100 bash[82644]: cluster 2024-07-28T19:39:48.061818+0000 mon.a (mon.0) 584 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-07-28T19:39:49.985 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:39:50.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: cluster 2024-07-28T19:39:49.071624+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-07-28T19:39:50.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.519984+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.528503+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.530592+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.532094+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.540509+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.652665+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.657047+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.659679+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.661823+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: audit 2024-07-28T19:39:49.664245+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[80199]: cluster 2024-07-28T19:39:49.838777+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-07-28T19:39:50.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: cluster 2024-07-28T19:39:49.071624+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.519984+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.528503+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.530592+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.532094+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.540509+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.652665+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.657047+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.659679+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.661823+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: audit 2024-07-28T19:39:49.664245+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-28T19:39:50.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:50 smithi100 bash[82644]: cluster 2024-07-28T19:39:49.838777+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-07-28T19:39:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: cluster 2024-07-28T19:39:49.071624+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-07-28T19:39:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.519984+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.528503+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.530592+0000 mon.a (mon.0) 588 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.532094+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:39:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.540509+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.652665+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:39:50.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.657047+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.659679+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.661823+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:39:50.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: audit 2024-07-28T19:39:49.664245+0000 mon.a (mon.0) 595 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-28T19:39:50.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:50 smithi138 bash[67894]: cluster 2024-07-28T19:39:49.838777+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-07-28T19:39:51.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:51 smithi100 bash[80199]: cluster 2024-07-28T19:39:49.637222+0000 mgr.y (mgr.25358) 193 : cluster [DBG] pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 5.2 KiB/s wr, 113 op/s 2024-07-28T19:39:51.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:51 smithi100 bash[80199]: audit 2024-07-28T19:39:49.664592+0000 mgr.y (mgr.25358) 194 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-28T19:39:51.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:51 smithi100 bash[80199]: cephadm 2024-07-28T19:39:49.666674+0000 mgr.y (mgr.25358) 195 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-07-28T19:39:51.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:51 smithi100 bash[80199]: cluster 2024-07-28T19:39:50.843999+0000 mon.a (mon.0) 597 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-07-28T19:39:51.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:51 smithi100 bash[82644]: cluster 2024-07-28T19:39:49.637222+0000 mgr.y (mgr.25358) 193 : cluster [DBG] pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 5.2 KiB/s wr, 113 op/s 2024-07-28T19:39:51.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:51 smithi100 bash[82644]: audit 2024-07-28T19:39:49.664592+0000 mgr.y (mgr.25358) 194 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-28T19:39:51.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:51 smithi100 bash[82644]: cephadm 2024-07-28T19:39:49.666674+0000 mgr.y (mgr.25358) 195 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-07-28T19:39:51.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:51 smithi100 bash[82644]: cluster 2024-07-28T19:39:50.843999+0000 mon.a (mon.0) 597 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-07-28T19:39:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:51 smithi138 bash[67894]: cluster 2024-07-28T19:39:49.637222+0000 mgr.y (mgr.25358) 193 : cluster [DBG] pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 5.2 KiB/s wr, 113 op/s 2024-07-28T19:39:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:51 smithi138 bash[67894]: audit 2024-07-28T19:39:49.664592+0000 mgr.y (mgr.25358) 194 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-07-28T19:39:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:51 smithi138 bash[67894]: cephadm 2024-07-28T19:39:49.666674+0000 mgr.y (mgr.25358) 195 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-07-28T19:39:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:51 smithi138 bash[67894]: cluster 2024-07-28T19:39:50.843999+0000 mon.a (mon.0) 597 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-07-28T19:39:52.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:39:52 smithi138 bash[60615]: ts=2024-07-28T19:39:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:39:53.169 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[82644]: cluster 2024-07-28T19:39:51.637718+0000 mgr.y (mgr.25358) 196 : cluster [DBG] pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 7.5 KiB/s wr, 135 op/s 2024-07-28T19:39:53.170 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[82644]: cluster 2024-07-28T19:39:51.842049+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-07-28T19:39:53.170 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[82644]: audit 2024-07-28T19:39:52.637242+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:53.170 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[82644]: audit 2024-07-28T19:39:52.639215+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:53.170 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[82644]: audit 2024-07-28T19:39:52.720251+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:53.171 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:39:52] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2024-07-28T19:39:53.171 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[80199]: cluster 2024-07-28T19:39:51.637718+0000 mgr.y (mgr.25358) 196 : cluster [DBG] pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 7.5 KiB/s wr, 135 op/s 2024-07-28T19:39:53.171 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[80199]: cluster 2024-07-28T19:39:51.842049+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-07-28T19:39:53.171 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[80199]: audit 2024-07-28T19:39:52.637242+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:53.171 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[80199]: audit 2024-07-28T19:39:52.639215+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:53.171 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:52 smithi100 bash[80199]: audit 2024-07-28T19:39:52.720251+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:52 smithi138 bash[67894]: cluster 2024-07-28T19:39:51.637718+0000 mgr.y (mgr.25358) 196 : cluster [DBG] pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 7.5 KiB/s wr, 135 op/s 2024-07-28T19:39:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:52 smithi138 bash[67894]: cluster 2024-07-28T19:39:51.842049+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-07-28T19:39:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:52 smithi138 bash[67894]: audit 2024-07-28T19:39:52.637242+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:52 smithi138 bash[67894]: audit 2024-07-28T19:39:52.639215+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:39:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:52 smithi138 bash[67894]: audit 2024-07-28T19:39:52.720251+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:53.993 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestMirroring.AioSnapshotCreate (33003 ms) 2024-07-28T19:39:53.995 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[82644]: cluster 2024-07-28T19:39:52.859744+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[82644]: cephadm 2024-07-28T19:39:53.068658+0000 mgr.y (mgr.25358) 197 : cephadm [INF] Upgrade: Updating osd.1 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[82644]: audit 2024-07-28T19:39:53.075233+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[82644]: audit 2024-07-28T19:39:53.076240+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[82644]: audit 2024-07-28T19:39:53.076986+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[82644]: cephadm 2024-07-28T19:39:53.081198+0000 mgr.y (mgr.25358) 198 : cephadm [INF] Deploying daemon osd.1 on smithi100 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[80199]: cluster 2024-07-28T19:39:52.859744+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[80199]: cephadm 2024-07-28T19:39:53.068658+0000 mgr.y (mgr.25358) 197 : cephadm [INF] Upgrade: Updating osd.1 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[80199]: audit 2024-07-28T19:39:53.075233+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[80199]: audit 2024-07-28T19:39:53.076240+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:39:53.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[80199]: audit 2024-07-28T19:39:53.076986+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:53.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:53 smithi100 bash[80199]: cephadm 2024-07-28T19:39:53.081198+0000 mgr.y (mgr.25358) 198 : cephadm [INF] Deploying daemon osd.1 on smithi100 2024-07-28T19:39:54.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:53 smithi138 bash[67894]: cluster 2024-07-28T19:39:52.859744+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-07-28T19:39:54.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:53 smithi138 bash[67894]: cephadm 2024-07-28T19:39:53.068658+0000 mgr.y (mgr.25358) 197 : cephadm [INF] Upgrade: Updating osd.1 2024-07-28T19:39:54.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:53 smithi138 bash[67894]: audit 2024-07-28T19:39:53.075233+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:54.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:53 smithi138 bash[67894]: audit 2024-07-28T19:39:53.076240+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-07-28T19:39:54.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:53 smithi138 bash[67894]: audit 2024-07-28T19:39:53.076986+0000 mon.a (mon.0) 605 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:39:54.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:53 smithi138 bash[67894]: cephadm 2024-07-28T19:39:53.081198+0000 mgr.y (mgr.25358) 198 : cephadm [INF] Deploying daemon osd.1 on smithi100 2024-07-28T19:39:54.845 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 51 tests from TestMirroring (94785 ms total) 2024-07-28T19:39:54.846 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:39:54.846 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 34 tests from TestInternal 2024-07-28T19:39:55.139 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[82644]: cluster 2024-07-28T19:39:53.638953+0000 mgr.y (mgr.25358) 199 : cluster [DBG] pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 8.7 KiB/s wr, 128 op/s 2024-07-28T19:39:55.139 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[82644]: cluster 2024-07-28T19:39:53.871892+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-07-28T19:39:55.139 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[82644]: audit 2024-07-28T19:39:53.980071+0000 mon.a (mon.0) 607 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/6c0942f6-7d75-4d19-8fa1-e72230ae905f"}]: dispatch 2024-07-28T19:39:55.139 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[82644]: audit 2024-07-28T19:39:53.986756+0000 mon.a (mon.0) 608 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/6c0942f6-7d75-4d19-8fa1-e72230ae905f"}]': finished 2024-07-28T19:39:55.139 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[82644]: cluster 2024-07-28T19:39:54.842526+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-07-28T19:39:55.140 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[80199]: cluster 2024-07-28T19:39:53.638953+0000 mgr.y (mgr.25358) 199 : cluster [DBG] pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 8.7 KiB/s wr, 128 op/s 2024-07-28T19:39:55.140 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[80199]: cluster 2024-07-28T19:39:53.871892+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-07-28T19:39:55.140 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[80199]: audit 2024-07-28T19:39:53.980071+0000 mon.a (mon.0) 607 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/6c0942f6-7d75-4d19-8fa1-e72230ae905f"}]: dispatch 2024-07-28T19:39:55.140 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[80199]: audit 2024-07-28T19:39:53.986756+0000 mon.a (mon.0) 608 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/6c0942f6-7d75-4d19-8fa1-e72230ae905f"}]': finished 2024-07-28T19:39:55.140 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:54 smithi100 bash[80199]: cluster 2024-07-28T19:39:54.842526+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-07-28T19:39:55.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:54 smithi138 bash[67894]: cluster 2024-07-28T19:39:53.638953+0000 mgr.y (mgr.25358) 199 : cluster [DBG] pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 88 KiB/s rd, 8.7 KiB/s wr, 128 op/s 2024-07-28T19:39:55.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:54 smithi138 bash[67894]: cluster 2024-07-28T19:39:53.871892+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-07-28T19:39:55.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:54 smithi138 bash[67894]: audit 2024-07-28T19:39:53.980071+0000 mon.a (mon.0) 607 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/6c0942f6-7d75-4d19-8fa1-e72230ae905f"}]: dispatch 2024-07-28T19:39:55.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:54 smithi138 bash[67894]: audit 2024-07-28T19:39:53.986756+0000 mon.a (mon.0) 608 : audit [INF] from='client.? 172.21.15.100:0/2486179613' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/6c0942f6-7d75-4d19-8fa1-e72230ae905f"}]': finished 2024-07-28T19:39:55.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:54 smithi138 bash[67894]: cluster 2024-07-28T19:39:54.842526+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-07-28T19:39:55.484 INFO:teuthology.orchestra.run.smithi100.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:39:56.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[80199]: audit 2024-07-28T19:39:55.482644+0000 mgr.y (mgr.25358) 200 : audit [DBG] from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:56.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[80199]: cephadm 2024-07-28T19:39:55.483225+0000 mgr.y (mgr.25358) 201 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:39:56.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[80199]: audit 2024-07-28T19:39:55.488435+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:56.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[80199]: cluster 2024-07-28T19:39:55.639649+0000 mgr.y (mgr.25358) 202 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:56.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[80199]: cluster 2024-07-28T19:39:55.876763+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-07-28T19:39:56.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[80199]: audit 2024-07-28T19:39:55.879883+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.100:0/3541727793' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:39:56.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[80199]: audit 2024-07-28T19:39:55.893517+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:39:56.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[82644]: audit 2024-07-28T19:39:55.482644+0000 mgr.y (mgr.25358) 200 : audit [DBG] from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:56.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[82644]: cephadm 2024-07-28T19:39:55.483225+0000 mgr.y (mgr.25358) 201 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:39:56.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[82644]: audit 2024-07-28T19:39:55.488435+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:56.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[82644]: cluster 2024-07-28T19:39:55.639649+0000 mgr.y (mgr.25358) 202 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:56.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[82644]: cluster 2024-07-28T19:39:55.876763+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-07-28T19:39:56.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[82644]: audit 2024-07-28T19:39:55.879883+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.100:0/3541727793' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:39:56.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:56 smithi100 bash[82644]: audit 2024-07-28T19:39:55.893517+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:39:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:56 smithi138 bash[67894]: audit 2024-07-28T19:39:55.482644+0000 mgr.y (mgr.25358) 200 : audit [DBG] from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:39:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:56 smithi138 bash[67894]: cephadm 2024-07-28T19:39:55.483225+0000 mgr.y (mgr.25358) 201 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:39:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:56 smithi138 bash[67894]: audit 2024-07-28T19:39:55.488435+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:39:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:56 smithi138 bash[67894]: cluster 2024-07-28T19:39:55.639649+0000 mgr.y (mgr.25358) 202 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:56 smithi138 bash[67894]: cluster 2024-07-28T19:39:55.876763+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-07-28T19:39:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:56 smithi138 bash[67894]: audit 2024-07-28T19:39:55.879883+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.100:0/3541727793' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:39:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:56 smithi138 bash[67894]: audit 2024-07-28T19:39:55.893517+0000 mon.a (mon.0) 612 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:39:56.882 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.OpenByID 2024-07-28T19:39:56.959 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'sleep 60' 2024-07-28T19:39:57.995 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:57 smithi100 bash[80199]: audit 2024-07-28T19:39:56.885179+0000 mon.a (mon.0) 613 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:39:57.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:57 smithi100 bash[80199]: cluster 2024-07-28T19:39:56.885792+0000 mon.a (mon.0) 614 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-07-28T19:39:57.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:57 smithi100 bash[82644]: audit 2024-07-28T19:39:56.885179+0000 mon.a (mon.0) 613 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:39:57.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:57 smithi100 bash[82644]: cluster 2024-07-28T19:39:56.885792+0000 mon.a (mon.0) 614 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-07-28T19:39:58.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:57 smithi138 bash[67894]: audit 2024-07-28T19:39:56.885179+0000 mon.a (mon.0) 613 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:39:58.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:57 smithi138 bash[67894]: cluster 2024-07-28T19:39:56.885792+0000 mon.a (mon.0) 614 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-07-28T19:39:58.927 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.OpenByID (2044 ms) 2024-07-28T19:39:58.927 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-07-28T19:39:58.968 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.OpenSnapDNE (42 ms) 2024-07-28T19:39:58.968 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-07-28T19:39:59.004 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (36 ms) 2024-07-28T19:39:59.004 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-07-28T19:39:59.052 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.ResizeLocksImage (48 ms) 2024-07-28T19:39:59.052 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-07-28T19:39:59.068 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.068 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.068 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.069 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.069 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.069 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:58 smithi100 bash[80199]: cluster 2024-07-28T19:39:57.640298+0000 mgr.y (mgr.25358) 203 : cluster [DBG] pgmap v352: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:59.069 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:58 smithi100 bash[80199]: cluster 2024-07-28T19:39:57.892029+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-07-28T19:39:59.070 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.070 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.070 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:58 smithi100 bash[82644]: cluster 2024-07-28T19:39:57.640298+0000 mgr.y (mgr.25358) 203 : cluster [DBG] pgmap v352: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:59.070 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:58 smithi100 bash[82644]: cluster 2024-07-28T19:39:57.892029+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-07-28T19:39:59.070 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.071 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:39:58 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:39:59.084 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (32 ms) 2024-07-28T19:39:59.084 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-07-28T19:39:59.305 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:58 smithi138 bash[67894]: cluster 2024-07-28T19:39:57.640298+0000 mgr.y (mgr.25358) 203 : cluster [DBG] pgmap v352: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:39:59.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:58 smithi138 bash[67894]: cluster 2024-07-28T19:39:57.892029+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2024-07-28T19:39:59.496 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:39:59 smithi100 systemd[1]: Stopping Ceph osd.1 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:39:59.496 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:39:59 smithi100 bash[40300]: debug 2024-07-28T19:39:59.204+0000 7fe3cbf06640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:39:59.496 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:39:59 smithi100 bash[40300]: debug 2024-07-28T19:39:59.204+0000 7fe3cbf06640 -1 osd.1 524 *** Got signal Terminated *** 2024-07-28T19:39:59.496 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:39:59 smithi100 bash[40300]: debug 2024-07-28T19:39:59.204+0000 7fe3cbf06640 -1 osd.1 524 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:40:00.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:59 smithi100 bash[82644]: cluster 2024-07-28T19:39:58.893931+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-07-28T19:40:00.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:39:59 smithi100 bash[82644]: cluster 2024-07-28T19:39:59.207826+0000 mon.a (mon.0) 617 : cluster [INF] osd.1 marked itself down and dead 2024-07-28T19:40:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:59 smithi100 bash[80199]: debug 2024-07-28T19:39:59.896+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:40:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:59 smithi100 bash[80199]: cluster 2024-07-28T19:39:58.893931+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-07-28T19:40:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:39:59 smithi100 bash[80199]: cluster 2024-07-28T19:39:59.207826+0000 mon.a (mon.0) 617 : cluster [INF] osd.1 marked itself down and dead 2024-07-28T19:40:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1 osds down; 7 osds(s) are not reachable 2024-07-28T19:40:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : [WRN] OSD_DOWN: 1 osds down 2024-07-28T19:40:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.1 (root=default,host=smithi100) is down 2024-07-28T19:40:00.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 7 osds(s) are not reachable 2024-07-28T19:40:00.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:00.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:00.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:00.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:00.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:00.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:00.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: debug 2024-07-28T19:39:59.996+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:00.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:59 smithi138 bash[67894]: cluster 2024-07-28T19:39:58.893931+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2024-07-28T19:40:00.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:39:59 smithi138 bash[67894]: cluster 2024-07-28T19:39:59.207826+0000 mon.a (mon.0) 617 : cluster [INF] osd.1 marked itself down and dead 2024-07-28T19:40:01.246 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:01 smithi100 bash[93962]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-1 2024-07-28T19:40:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:39:59.640936+0000 mgr.y (mgr.25358) 204 : cluster [DBG] pgmap v355: 105 pgs: 7 unknown, 98 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:40:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:39:59.898326+0000 mon.a (mon.0) 618 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:39:59.898378+0000 mon.a (mon.0) 619 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:39:59.913691+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e525: 8 total, 7 up, 8 in 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000173+0000 mon.a (mon.0) 621 : cluster [ERR] Health detail: HEALTH_ERR 1 osds down; 7 osds(s) are not reachable 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000214+0000 mon.a (mon.0) 622 : cluster [ERR] [WRN] OSD_DOWN: 1 osds down 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000252+0000 mon.a (mon.0) 623 : cluster [ERR] osd.1 (root=default,host=smithi100) is down 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000287+0000 mon.a (mon.0) 624 : cluster [ERR] [ERR] OSD_UNREACHABLE: 7 osds(s) are not reachable 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000322+0000 mon.a (mon.0) 625 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000353+0000 mon.a (mon.0) 626 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000403+0000 mon.a (mon.0) 627 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000439+0000 mon.a (mon.0) 628 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000470+0000 mon.a (mon.0) 629 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000498+0000 mon.a (mon.0) 630 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.000536+0000 mon.a (mon.0) 631 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:39:59.640936+0000 mgr.y (mgr.25358) 204 : cluster [DBG] pgmap v355: 105 pgs: 7 unknown, 98 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:39:59.898326+0000 mon.a (mon.0) 618 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:39:59.898378+0000 mon.a (mon.0) 619 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:39:59.913691+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e525: 8 total, 7 up, 8 in 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000173+0000 mon.a (mon.0) 621 : cluster [ERR] Health detail: HEALTH_ERR 1 osds down; 7 osds(s) are not reachable 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000214+0000 mon.a (mon.0) 622 : cluster [ERR] [WRN] OSD_DOWN: 1 osds down 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000252+0000 mon.a (mon.0) 623 : cluster [ERR] osd.1 (root=default,host=smithi100) is down 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000287+0000 mon.a (mon.0) 624 : cluster [ERR] [ERR] OSD_UNREACHABLE: 7 osds(s) are not reachable 2024-07-28T19:40:01.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000322+0000 mon.a (mon.0) 625 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000353+0000 mon.a (mon.0) 626 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000403+0000 mon.a (mon.0) 627 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000439+0000 mon.a (mon.0) 628 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000470+0000 mon.a (mon.0) 629 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000498+0000 mon.a (mon.0) 630 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.000536+0000 mon.a (mon.0) 631 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:39:59.640936+0000 mgr.y (mgr.25358) 204 : cluster [DBG] pgmap v355: 105 pgs: 7 unknown, 98 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:40:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:39:59.898326+0000 mon.a (mon.0) 618 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:40:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:39:59.898378+0000 mon.a (mon.0) 619 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:40:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:39:59.913691+0000 mon.a (mon.0) 620 : cluster [DBG] osdmap e525: 8 total, 7 up, 8 in 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000173+0000 mon.a (mon.0) 621 : cluster [ERR] Health detail: HEALTH_ERR 1 osds down; 7 osds(s) are not reachable 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000214+0000 mon.a (mon.0) 622 : cluster [ERR] [WRN] OSD_DOWN: 1 osds down 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000252+0000 mon.a (mon.0) 623 : cluster [ERR] osd.1 (root=default,host=smithi100) is down 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000287+0000 mon.a (mon.0) 624 : cluster [ERR] [ERR] OSD_UNREACHABLE: 7 osds(s) are not reachable 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000322+0000 mon.a (mon.0) 625 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000353+0000 mon.a (mon.0) 626 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000403+0000 mon.a (mon.0) 627 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000439+0000 mon.a (mon.0) 628 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000470+0000 mon.a (mon.0) 629 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000498+0000 mon.a (mon.0) 630 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.000536+0000 mon.a (mon.0) 631 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:40:01.781 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:01 smithi100 bash[94044]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-1 2024-07-28T19:40:01.960 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SnapCreateLocksImage (2875 ms) 2024-07-28T19:40:01.960 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-07-28T19:40:02.014 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (55 ms) 2024-07-28T19:40:02.015 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-07-28T19:40:02.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:01 smithi100 bash[80199]: cluster 2024-07-28T19:40:00.927309+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e526: 8 total, 7 up, 8 in 2024-07-28T19:40:02.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:01 smithi100 bash[82644]: cluster 2024-07-28T19:40:00.927309+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e526: 8 total, 7 up, 8 in 2024-07-28T19:40:02.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:01 smithi138 bash[67894]: cluster 2024-07-28T19:40:00.927309+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e526: 8 total, 7 up, 8 in 2024-07-28T19:40:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:40:02 smithi138 bash[60615]: ts=2024-07-28T19:40:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:40:02.812 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:40:02.995 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:40:02] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2024-07-28T19:40:02.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:02 smithi100 bash[82644]: cluster 2024-07-28T19:40:01.641652+0000 mgr.y (mgr.25358) 205 : cluster [DBG] pgmap v358: 105 pgs: 6 peering, 11 stale+active+clean, 88 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 1.2 KiB/s wr, 18 op/s 2024-07-28T19:40:02.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:02 smithi100 bash[82644]: cluster 2024-07-28T19:40:01.940721+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e527: 8 total, 7 up, 8 in 2024-07-28T19:40:02.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:02 smithi100 bash[80199]: cluster 2024-07-28T19:40:01.641652+0000 mgr.y (mgr.25358) 205 : cluster [DBG] pgmap v358: 105 pgs: 6 peering, 11 stale+active+clean, 88 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 1.2 KiB/s wr, 18 op/s 2024-07-28T19:40:02.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:02 smithi100 bash[80199]: cluster 2024-07-28T19:40:01.940721+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e527: 8 total, 7 up, 8 in 2024-07-28T19:40:03.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:02 smithi138 bash[67894]: cluster 2024-07-28T19:40:01.641652+0000 mgr.y (mgr.25358) 205 : cluster [DBG] pgmap v358: 105 pgs: 6 peering, 11 stale+active+clean, 88 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 1.2 KiB/s wr, 18 op/s 2024-07-28T19:40:03.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:02 smithi138 bash[67894]: cluster 2024-07-28T19:40:01.940721+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e527: 8 total, 7 up, 8 in 2024-07-28T19:40:03.980 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (1965 ms) 2024-07-28T19:40:03.980 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-07-28T19:40:04.197 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:03 smithi100 bash[80199]: cluster 2024-07-28T19:40:02.939358+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e528: 8 total, 7 up, 8 in 2024-07-28T19:40:04.198 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.1.service: Deactivated successfully. 2024-07-28T19:40:04.198 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: Stopped Ceph osd.1 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:40:04.198 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:03 smithi100 bash[82644]: cluster 2024-07-28T19:40:02.939358+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e528: 8 total, 7 up, 8 in 2024-07-28T19:40:04.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:03 smithi138 bash[67894]: cluster 2024-07-28T19:40:02.939358+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e528: 8 total, 7 up, 8 in 2024-07-28T19:40:04.463 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.464 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.464 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.464 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.465 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.465 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.465 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.465 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.466 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:40:04.745 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:04 smithi100 systemd[1]: Started Ceph osd.1 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:40:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[80199]: cluster 2024-07-28T19:40:03.643065+0000 mgr.y (mgr.25358) 206 : cluster [DBG] pgmap v361: 105 pgs: 34 active+undersized, 6 peering, 9 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 11 KiB/s wr, 138 op/s; 13/144 objects degraded (9.028%) 2024-07-28T19:40:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[80199]: cluster 2024-07-28T19:40:03.941386+0000 mon.a (mon.0) 635 : cluster [WRN] Health check failed: Degraded data redundancy: 13/144 objects degraded (9.028%), 9 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[80199]: cluster 2024-07-28T19:40:03.955898+0000 mon.a (mon.0) 636 : cluster [DBG] osdmap e529: 8 total, 7 up, 8 in 2024-07-28T19:40:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[80199]: audit 2024-07-28T19:40:04.506331+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[80199]: audit 2024-07-28T19:40:04.512804+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[80199]: audit 2024-07-28T19:40:04.514028+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:05.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[80199]: cluster 2024-07-28T19:40:04.843793+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2024-07-28T19:40:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[82644]: cluster 2024-07-28T19:40:03.643065+0000 mgr.y (mgr.25358) 206 : cluster [DBG] pgmap v361: 105 pgs: 34 active+undersized, 6 peering, 9 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 11 KiB/s wr, 138 op/s; 13/144 objects degraded (9.028%) 2024-07-28T19:40:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[82644]: cluster 2024-07-28T19:40:03.941386+0000 mon.a (mon.0) 635 : cluster [WRN] Health check failed: Degraded data redundancy: 13/144 objects degraded (9.028%), 9 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[82644]: cluster 2024-07-28T19:40:03.955898+0000 mon.a (mon.0) 636 : cluster [DBG] osdmap e529: 8 total, 7 up, 8 in 2024-07-28T19:40:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[82644]: audit 2024-07-28T19:40:04.506331+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[82644]: audit 2024-07-28T19:40:04.512804+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[82644]: audit 2024-07-28T19:40:04.514028+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:04 smithi100 bash[82644]: cluster 2024-07-28T19:40:04.843793+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2024-07-28T19:40:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:04 smithi138 bash[67894]: cluster 2024-07-28T19:40:03.643065+0000 mgr.y (mgr.25358) 206 : cluster [DBG] pgmap v361: 105 pgs: 34 active+undersized, 6 peering, 9 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s rd, 11 KiB/s wr, 138 op/s; 13/144 objects degraded (9.028%) 2024-07-28T19:40:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:04 smithi138 bash[67894]: cluster 2024-07-28T19:40:03.941386+0000 mon.a (mon.0) 635 : cluster [WRN] Health check failed: Degraded data redundancy: 13/144 objects degraded (9.028%), 9 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:04 smithi138 bash[67894]: cluster 2024-07-28T19:40:03.955898+0000 mon.a (mon.0) 636 : cluster [DBG] osdmap e529: 8 total, 7 up, 8 in 2024-07-28T19:40:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:04 smithi138 bash[67894]: audit 2024-07-28T19:40:04.506331+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:04 smithi138 bash[67894]: audit 2024-07-28T19:40:04.512804+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:04 smithi138 bash[67894]: audit 2024-07-28T19:40:04.514028+0000 mon.a (mon.0) 639 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:04 smithi138 bash[67894]: cluster 2024-07-28T19:40:04.843793+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2024-07-28T19:40:05.884 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (1904 ms) 2024-07-28T19:40:05.884 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-07-28T19:40:07.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:06 smithi100 bash[82644]: cluster 2024-07-28T19:40:05.644089+0000 mgr.y (mgr.25358) 207 : cluster [DBG] pgmap v364: 105 pgs: 37 active+undersized, 12 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 11 KiB/s wr, 136 op/s; 18/153 objects degraded (11.765%) 2024-07-28T19:40:07.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:06 smithi100 bash[82644]: cluster 2024-07-28T19:40:05.860954+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2024-07-28T19:40:07.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:06 smithi100 bash[80199]: cluster 2024-07-28T19:40:05.644089+0000 mgr.y (mgr.25358) 207 : cluster [DBG] pgmap v364: 105 pgs: 37 active+undersized, 12 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 11 KiB/s wr, 136 op/s; 18/153 objects degraded (11.765%) 2024-07-28T19:40:07.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:06 smithi100 bash[80199]: cluster 2024-07-28T19:40:05.860954+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2024-07-28T19:40:07.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:06 smithi138 bash[67894]: cluster 2024-07-28T19:40:05.644089+0000 mgr.y (mgr.25358) 207 : cluster [DBG] pgmap v364: 105 pgs: 37 active+undersized, 12 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s rd, 11 KiB/s wr, 136 op/s; 18/153 objects degraded (11.765%) 2024-07-28T19:40:07.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:06 smithi138 bash[67894]: cluster 2024-07-28T19:40:05.860954+0000 mon.a (mon.0) 641 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2024-07-28T19:40:07.604 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[94242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-07-28T19:40:07.605 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[94242]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-07-28T19:40:07.605 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[94242]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-07-28T19:40:07.605 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[94242]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-07-28T19:40:07.605 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[94242]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-07-28T19:40:07.605 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[94242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-07-28T19:40:07.605 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[94242]: --> ceph-volume raw activate successful for osd ID: 1 2024-07-28T19:40:07.992 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SnapSetReleasesLock (2109 ms) 2024-07-28T19:40:07.993 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-07-28T19:40:08.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[82644]: cluster 2024-07-28T19:40:06.951162+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2024-07-28T19:40:08.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[82644]: audit 2024-07-28T19:40:07.639995+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:08.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[82644]: audit 2024-07-28T19:40:07.641907+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[80199]: cluster 2024-07-28T19:40:06.951162+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2024-07-28T19:40:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[80199]: audit 2024-07-28T19:40:07.639995+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:08.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:07 smithi100 bash[80199]: audit 2024-07-28T19:40:07.641907+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:07 smithi138 bash[67894]: cluster 2024-07-28T19:40:06.951162+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2024-07-28T19:40:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:07 smithi138 bash[67894]: audit 2024-07-28T19:40:07.639995+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:07 smithi138 bash[67894]: audit 2024-07-28T19:40:07.641907+0000 mon.a (mon.0) 644 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:08 smithi100 bash[82644]: cluster 2024-07-28T19:40:07.644834+0000 mgr.y (mgr.25358) 208 : cluster [DBG] pgmap v367: 105 pgs: 37 active+undersized, 12 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 1023 B/s wr, 16 op/s; 18/153 objects degraded (11.765%) 2024-07-28T19:40:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:08 smithi100 bash[82644]: cluster 2024-07-28T19:40:07.970390+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2024-07-28T19:40:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:08 smithi100 bash[80199]: cluster 2024-07-28T19:40:07.644834+0000 mgr.y (mgr.25358) 208 : cluster [DBG] pgmap v367: 105 pgs: 37 active+undersized, 12 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 1023 B/s wr, 16 op/s; 18/153 objects degraded (11.765%) 2024-07-28T19:40:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:08 smithi100 bash[80199]: cluster 2024-07-28T19:40:07.970390+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2024-07-28T19:40:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:08 smithi138 bash[67894]: cluster 2024-07-28T19:40:07.644834+0000 mgr.y (mgr.25358) 208 : cluster [DBG] pgmap v367: 105 pgs: 37 active+undersized, 12 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 1023 B/s wr, 16 op/s; 18/153 objects degraded (11.765%) 2024-07-28T19:40:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:08 smithi138 bash[67894]: cluster 2024-07-28T19:40:07.970390+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2024-07-28T19:40:09.746 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:09 smithi100 bash[94549]: debug 2024-07-28T19:40:09.492+0000 7efe1daca640 1 -- 172.21.15.100:0/3209164705 <== mon.0 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ca1510f1e0 con 0x55ca15e8c000 2024-07-28T19:40:10.025 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.FlattenLocksImage (2032 ms) 2024-07-28T19:40:10.025 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-07-28T19:40:10.246 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:10 smithi100 bash[94549]: debug 2024-07-28T19:40:10.044+0000 7efe20330640 -1 Falling back to public interface 2024-07-28T19:40:10.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:09 smithi100 bash[80199]: cluster 2024-07-28T19:40:08.970830+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2024-07-28T19:40:10.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:09 smithi100 bash[80199]: cluster 2024-07-28T19:40:09.839660+0000 mon.a (mon.0) 647 : cluster [WRN] Health check update: Degraded data redundancy: 18/153 objects degraded (11.765%), 12 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:10.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:09 smithi100 bash[82644]: cluster 2024-07-28T19:40:08.970830+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2024-07-28T19:40:10.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:09 smithi100 bash[82644]: cluster 2024-07-28T19:40:09.839660+0000 mon.a (mon.0) 647 : cluster [WRN] Health check update: Degraded data redundancy: 18/153 objects degraded (11.765%), 12 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:09 smithi138 bash[67894]: cluster 2024-07-28T19:40:08.970830+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2024-07-28T19:40:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:09 smithi138 bash[67894]: cluster 2024-07-28T19:40:09.839660+0000 mon.a (mon.0) 647 : cluster [WRN] Health check update: Degraded data redundancy: 18/153 objects degraded (11.765%), 12 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:11.245 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:10 smithi100 bash[80199]: cluster 2024-07-28T19:40:09.645995+0000 mgr.y (mgr.25358) 209 : cluster [DBG] pgmap v370: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 6.2 KiB/s wr, 88 op/s; 24/174 objects degraded (13.793%) 2024-07-28T19:40:11.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:10 smithi100 bash[80199]: cluster 2024-07-28T19:40:09.993258+0000 mon.a (mon.0) 648 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-07-28T19:40:11.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:10 smithi100 bash[82644]: cluster 2024-07-28T19:40:09.645995+0000 mgr.y (mgr.25358) 209 : cluster [DBG] pgmap v370: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 6.2 KiB/s wr, 88 op/s; 24/174 objects degraded (13.793%) 2024-07-28T19:40:11.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:10 smithi100 bash[82644]: cluster 2024-07-28T19:40:09.993258+0000 mon.a (mon.0) 648 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-07-28T19:40:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:10 smithi138 bash[67894]: cluster 2024-07-28T19:40:09.645995+0000 mgr.y (mgr.25358) 209 : cluster [DBG] pgmap v370: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 6.2 KiB/s wr, 88 op/s; 24/174 objects degraded (13.793%) 2024-07-28T19:40:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:10 smithi138 bash[67894]: cluster 2024-07-28T19:40:09.993258+0000 mon.a (mon.0) 648 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-07-28T19:40:12.023 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (1999 ms) 2024-07-28T19:40:12.023 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-07-28T19:40:12.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:12 smithi138 bash[67894]: cluster 2024-07-28T19:40:11.001711+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-07-28T19:40:12.495 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:12 smithi100 bash[80199]: cluster 2024-07-28T19:40:11.001711+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-07-28T19:40:12.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:12 smithi100 bash[82644]: cluster 2024-07-28T19:40:11.001711+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-07-28T19:40:12.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:40:12 smithi138 bash[60615]: ts=2024-07-28T19:40:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:40:13.017 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (993 ms) 2024-07-28T19:40:13.017 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-07-28T19:40:13.187 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:40:12] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2024-07-28T19:40:13.187 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[80199]: cluster 2024-07-28T19:40:11.646796+0000 mgr.y (mgr.25358) 210 : cluster [DBG] pgmap v373: 105 pgs: 33 active+undersized, 16 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.0 KiB/s wr, 110 op/s; 25/180 objects degraded (13.889%) 2024-07-28T19:40:13.187 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[80199]: cluster 2024-07-28T19:40:12.006601+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-07-28T19:40:13.187 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[80199]: audit 2024-07-28T19:40:12.080819+0000 mon.a (mon.0) 651 : audit [INF] from='client.? 172.21.15.100:0/4293227232' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/4293227232"}]: dispatch 2024-07-28T19:40:13.187 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[80199]: audit 2024-07-28T19:40:12.504457+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:13.187 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[80199]: audit 2024-07-28T19:40:12.510565+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:13.188 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[82644]: cluster 2024-07-28T19:40:11.646796+0000 mgr.y (mgr.25358) 210 : cluster [DBG] pgmap v373: 105 pgs: 33 active+undersized, 16 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.0 KiB/s wr, 110 op/s; 25/180 objects degraded (13.889%) 2024-07-28T19:40:13.188 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[82644]: cluster 2024-07-28T19:40:12.006601+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-07-28T19:40:13.188 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[82644]: audit 2024-07-28T19:40:12.080819+0000 mon.a (mon.0) 651 : audit [INF] from='client.? 172.21.15.100:0/4293227232' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/4293227232"}]: dispatch 2024-07-28T19:40:13.188 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[82644]: audit 2024-07-28T19:40:12.504457+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:13.188 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[82644]: audit 2024-07-28T19:40:12.510565+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:13 smithi138 bash[67894]: cluster 2024-07-28T19:40:11.646796+0000 mgr.y (mgr.25358) 210 : cluster [DBG] pgmap v373: 105 pgs: 33 active+undersized, 16 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 7.0 KiB/s wr, 110 op/s; 25/180 objects degraded (13.889%) 2024-07-28T19:40:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:13 smithi138 bash[67894]: cluster 2024-07-28T19:40:12.006601+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-07-28T19:40:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:13 smithi138 bash[67894]: audit 2024-07-28T19:40:12.080819+0000 mon.a (mon.0) 651 : audit [INF] from='client.? 172.21.15.100:0/4293227232' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/4293227232"}]: dispatch 2024-07-28T19:40:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:13 smithi138 bash[67894]: audit 2024-07-28T19:40:12.504457+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:13 smithi138 bash[67894]: audit 2024-07-28T19:40:12.510565+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:13.495 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:13 smithi100 bash[94549]: debug 2024-07-28T19:40:13.188+0000 7efe20330640 -1 osd.1 524 log_to_monitors true 2024-07-28T19:40:14.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:14 smithi138 bash[67894]: audit 2024-07-28T19:40:13.009931+0000 mon.a (mon.0) 654 : audit [INF] from='client.? 172.21.15.100:0/4293227232' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/4293227232"}]': finished 2024-07-28T19:40:14.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:14 smithi138 bash[67894]: cluster 2024-07-28T19:40:13.009972+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-07-28T19:40:14.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:14 smithi138 bash[67894]: audit 2024-07-28T19:40:13.071097+0000 mon.a (mon.0) 656 : audit [INF] from='client.? 172.21.15.100:0/3094956043' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/3094956043"}]: dispatch 2024-07-28T19:40:14.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:14 smithi138 bash[67894]: audit 2024-07-28T19:40:13.195143+0000 mon.a (mon.0) 657 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:40:14.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:14 smithi138 bash[67894]: audit 2024-07-28T19:40:13.218039+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:14.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:14 smithi138 bash[67894]: audit 2024-07-28T19:40:13.223703+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:14.496 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[94549]: debug 2024-07-28T19:40:14.060+0000 7efe180da640 -1 osd.1 524 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-28T19:40:14.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[82644]: audit 2024-07-28T19:40:13.009931+0000 mon.a (mon.0) 654 : audit [INF] from='client.? 172.21.15.100:0/4293227232' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/4293227232"}]': finished 2024-07-28T19:40:14.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[82644]: cluster 2024-07-28T19:40:13.009972+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-07-28T19:40:14.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[82644]: audit 2024-07-28T19:40:13.071097+0000 mon.a (mon.0) 656 : audit [INF] from='client.? 172.21.15.100:0/3094956043' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/3094956043"}]: dispatch 2024-07-28T19:40:14.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[82644]: audit 2024-07-28T19:40:13.195143+0000 mon.a (mon.0) 657 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:40:14.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[82644]: audit 2024-07-28T19:40:13.218039+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:14.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[82644]: audit 2024-07-28T19:40:13.223703+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:14.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[80199]: audit 2024-07-28T19:40:13.009931+0000 mon.a (mon.0) 654 : audit [INF] from='client.? 172.21.15.100:0/4293227232' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/4293227232"}]': finished 2024-07-28T19:40:14.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[80199]: cluster 2024-07-28T19:40:13.009972+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-07-28T19:40:14.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[80199]: audit 2024-07-28T19:40:13.071097+0000 mon.a (mon.0) 656 : audit [INF] from='client.? 172.21.15.100:0/3094956043' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/3094956043"}]: dispatch 2024-07-28T19:40:14.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[80199]: audit 2024-07-28T19:40:13.195143+0000 mon.a (mon.0) 657 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-07-28T19:40:14.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[80199]: audit 2024-07-28T19:40:13.218039+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:14.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:14 smithi100 bash[80199]: audit 2024-07-28T19:40:13.223703+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:15.025 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (2008 ms) 2024-07-28T19:40:15.025 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-07-28T19:40:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:15 smithi138 bash[67894]: cluster 2024-07-28T19:40:13.648119+0000 mgr.y (mgr.25358) 211 : cluster [DBG] pgmap v376: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 185 KiB/s rd, 11 KiB/s wr, 245 op/s; 30/207 objects degraded (14.493%) 2024-07-28T19:40:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:15 smithi138 bash[67894]: audit 2024-07-28T19:40:14.023044+0000 mon.a (mon.0) 660 : audit [INF] from='client.? 172.21.15.100:0/3094956043' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/3094956043"}]': finished 2024-07-28T19:40:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:15 smithi138 bash[67894]: audit 2024-07-28T19:40:14.023261+0000 mon.a (mon.0) 661 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-28T19:40:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:15 smithi138 bash[67894]: cluster 2024-07-28T19:40:14.023411+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2024-07-28T19:40:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:15 smithi138 bash[67894]: audit 2024-07-28T19:40:14.024919+0000 mon.a (mon.0) 663 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:40:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:15 smithi138 bash[67894]: cluster 2024-07-28T19:40:14.840525+0000 mon.a (mon.0) 664 : cluster [WRN] Health check update: Degraded data redundancy: 30/207 objects degraded (14.493%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:15.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[82644]: cluster 2024-07-28T19:40:13.648119+0000 mgr.y (mgr.25358) 211 : cluster [DBG] pgmap v376: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 185 KiB/s rd, 11 KiB/s wr, 245 op/s; 30/207 objects degraded (14.493%) 2024-07-28T19:40:15.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[82644]: audit 2024-07-28T19:40:14.023044+0000 mon.a (mon.0) 660 : audit [INF] from='client.? 172.21.15.100:0/3094956043' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/3094956043"}]': finished 2024-07-28T19:40:15.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[82644]: audit 2024-07-28T19:40:14.023261+0000 mon.a (mon.0) 661 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-28T19:40:15.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[82644]: cluster 2024-07-28T19:40:14.023411+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2024-07-28T19:40:15.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[82644]: audit 2024-07-28T19:40:14.024919+0000 mon.a (mon.0) 663 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:40:15.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[82644]: cluster 2024-07-28T19:40:14.840525+0000 mon.a (mon.0) 664 : cluster [WRN] Health check update: Degraded data redundancy: 30/207 objects degraded (14.493%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[80199]: debug 2024-07-28T19:40:15.024+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:40:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[80199]: cluster 2024-07-28T19:40:13.648119+0000 mgr.y (mgr.25358) 211 : cluster [DBG] pgmap v376: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 185 KiB/s rd, 11 KiB/s wr, 245 op/s; 30/207 objects degraded (14.493%) 2024-07-28T19:40:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[80199]: audit 2024-07-28T19:40:14.023044+0000 mon.a (mon.0) 660 : audit [INF] from='client.? 172.21.15.100:0/3094956043' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/3094956043"}]': finished 2024-07-28T19:40:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[80199]: audit 2024-07-28T19:40:14.023261+0000 mon.a (mon.0) 661 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-07-28T19:40:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[80199]: cluster 2024-07-28T19:40:14.023411+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2024-07-28T19:40:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[80199]: audit 2024-07-28T19:40:14.024919+0000 mon.a (mon.0) 663 : audit [INF] from='osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:40:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:15 smithi100 bash[80199]: cluster 2024-07-28T19:40:14.840525+0000 mon.a (mon.0) 664 : cluster [WRN] Health check update: Degraded data redundancy: 30/207 objects degraded (14.493%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:16.124 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.AioWriteRequestsLock (1100 ms) 2024-07-28T19:40:16.124 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-07-28T19:40:16.156 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (32 ms) 2024-07-28T19:40:16.157 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-07-28T19:40:16.215 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.CancelAsyncResize (59 ms) 2024-07-28T19:40:16.215 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.MultipleResize 2024-07-28T19:40:16.327 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.MultipleResize (112 ms) 2024-07-28T19:40:16.327 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.Metadata 2024-07-28T19:40:16.410 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.Metadata (80 ms) 2024-07-28T19:40:16.410 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-07-28T19:40:16.483 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.MetadataConfApply (76 ms) 2024-07-28T19:40:16.483 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-07-28T19:40:16.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[82644]: cluster 2024-07-28T19:40:15.027059+0000 mon.a (mon.0) 665 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:40:16.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[82644]: cluster 2024-07-28T19:40:15.027100+0000 mon.a (mon.0) 666 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:40:16.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[82644]: cluster 2024-07-28T19:40:15.040412+0000 mon.a (mon.0) 667 : cluster [INF] osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357] boot 2024-07-28T19:40:16.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[82644]: cluster 2024-07-28T19:40:15.040505+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-07-28T19:40:16.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[82644]: audit 2024-07-28T19:40:15.041415+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:40:16.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[82644]: cluster 2024-07-28T19:40:16.040009+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-07-28T19:40:16.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[80199]: cluster 2024-07-28T19:40:15.027059+0000 mon.a (mon.0) 665 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:40:16.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[80199]: cluster 2024-07-28T19:40:15.027100+0000 mon.a (mon.0) 666 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:40:16.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[80199]: cluster 2024-07-28T19:40:15.040412+0000 mon.a (mon.0) 667 : cluster [INF] osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357] boot 2024-07-28T19:40:16.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[80199]: cluster 2024-07-28T19:40:15.040505+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-07-28T19:40:16.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[80199]: audit 2024-07-28T19:40:15.041415+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:40:16.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:16 smithi100 bash[80199]: cluster 2024-07-28T19:40:16.040009+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-07-28T19:40:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:16 smithi138 bash[67894]: cluster 2024-07-28T19:40:15.027059+0000 mon.a (mon.0) 665 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:40:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:16 smithi138 bash[67894]: cluster 2024-07-28T19:40:15.027100+0000 mon.a (mon.0) 666 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:40:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:16 smithi138 bash[67894]: cluster 2024-07-28T19:40:15.040412+0000 mon.a (mon.0) 667 : cluster [INF] osd.1 [v2:172.21.15.100:6810/1736817357,v1:172.21.15.100:6811/1736817357] boot 2024-07-28T19:40:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:16 smithi138 bash[67894]: cluster 2024-07-28T19:40:15.040505+0000 mon.a (mon.0) 668 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-07-28T19:40:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:16 smithi138 bash[67894]: audit 2024-07-28T19:40:15.041415+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-07-28T19:40:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:16 smithi138 bash[67894]: cluster 2024-07-28T19:40:16.040009+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-07-28T19:40:17.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:17 smithi100 bash[80199]: cluster 2024-07-28T19:40:15.649002+0000 mgr.y (mgr.25358) 212 : cluster [DBG] pgmap v379: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 11 KiB/s wr, 230 op/s; 32/219 objects degraded (14.612%) 2024-07-28T19:40:17.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:17 smithi100 bash[82644]: cluster 2024-07-28T19:40:15.649002+0000 mgr.y (mgr.25358) 212 : cluster [DBG] pgmap v379: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 11 KiB/s wr, 230 op/s; 32/219 objects degraded (14.612%) 2024-07-28T19:40:17.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:17 smithi138 bash[67894]: cluster 2024-07-28T19:40:15.649002+0000 mgr.y (mgr.25358) 212 : cluster [DBG] pgmap v379: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 11 KiB/s wr, 230 op/s; 32/219 objects degraded (14.612%) 2024-07-28T19:40:18.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:18 smithi100 bash[80199]: cluster 2024-07-28T19:40:17.087510+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-07-28T19:40:18.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:18 smithi100 bash[82644]: cluster 2024-07-28T19:40:17.087510+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-07-28T19:40:18.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:18 smithi138 bash[67894]: cluster 2024-07-28T19:40:17.087510+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-07-28T19:40:19.302 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SnapshotCopyup (2819 ms) 2024-07-28T19:40:19.302 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-07-28T19:40:19.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:19 smithi100 bash[80199]: cluster 2024-07-28T19:40:17.649833+0000 mgr.y (mgr.25358) 213 : cluster [DBG] pgmap v382: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.2 KiB/s wr, 6 op/s; 32/219 objects degraded (14.612%) 2024-07-28T19:40:19.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:19 smithi100 bash[80199]: cluster 2024-07-28T19:40:18.102740+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-07-28T19:40:19.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:19 smithi100 bash[82644]: cluster 2024-07-28T19:40:17.649833+0000 mgr.y (mgr.25358) 213 : cluster [DBG] pgmap v382: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.2 KiB/s wr, 6 op/s; 32/219 objects degraded (14.612%) 2024-07-28T19:40:19.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:19 smithi100 bash[82644]: cluster 2024-07-28T19:40:18.102740+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-07-28T19:40:19.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:19 smithi138 bash[67894]: cluster 2024-07-28T19:40:17.649833+0000 mgr.y (mgr.25358) 213 : cluster [DBG] pgmap v382: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.2 KiB/s wr, 6 op/s; 32/219 objects degraded (14.612%) 2024-07-28T19:40:19.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:19 smithi138 bash[67894]: cluster 2024-07-28T19:40:18.102740+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-07-28T19:40:20.439 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:20 smithi100 bash[82644]: cluster 2024-07-28T19:40:19.114581+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-07-28T19:40:20.440 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:20 smithi100 bash[82644]: cluster 2024-07-28T19:40:19.841751+0000 mon.a (mon.0) 674 : cluster [WRN] Health check update: Degraded data redundancy: 32/219 objects degraded (14.612%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:20.440 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:20 smithi100 bash[80199]: cluster 2024-07-28T19:40:19.114581+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-07-28T19:40:20.440 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:20 smithi100 bash[80199]: cluster 2024-07-28T19:40:19.841751+0000 mon.a (mon.0) 674 : cluster [WRN] Health check update: Degraded data redundancy: 32/219 objects degraded (14.612%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:20.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:20 smithi138 bash[67894]: cluster 2024-07-28T19:40:19.114581+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-07-28T19:40:20.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:20 smithi138 bash[67894]: cluster 2024-07-28T19:40:19.841751+0000 mon.a (mon.0) 674 : cluster [WRN] Health check update: Degraded data redundancy: 32/219 objects degraded (14.612%), 17 pgs degraded (PG_DEGRADED) 2024-07-28T19:40:21.192 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (1890 ms) 2024-07-28T19:40:21.192 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-07-28T19:40:21.453 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:21 smithi100 bash[80199]: cluster 2024-07-28T19:40:19.650269+0000 mgr.y (mgr.25358) 214 : cluster [DBG] pgmap v385: 105 pgs: 1 active+undersized, 4 active+undersized+degraded, 100 active+clean; 585 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 24 KiB/s wr, 211 op/s; 8/294 objects degraded (2.721%); 24 B/s, 21 keys/s, 4 objects/s recovering 2024-07-28T19:40:21.453 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:21 smithi100 bash[80199]: cluster 2024-07-28T19:40:20.125415+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-07-28T19:40:21.453 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:21 smithi100 bash[82644]: cluster 2024-07-28T19:40:19.650269+0000 mgr.y (mgr.25358) 214 : cluster [DBG] pgmap v385: 105 pgs: 1 active+undersized, 4 active+undersized+degraded, 100 active+clean; 585 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 24 KiB/s wr, 211 op/s; 8/294 objects degraded (2.721%); 24 B/s, 21 keys/s, 4 objects/s recovering 2024-07-28T19:40:21.453 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:21 smithi100 bash[82644]: cluster 2024-07-28T19:40:20.125415+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-07-28T19:40:21.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:21 smithi138 bash[67894]: cluster 2024-07-28T19:40:19.650269+0000 mgr.y (mgr.25358) 214 : cluster [DBG] pgmap v385: 105 pgs: 1 active+undersized, 4 active+undersized+degraded, 100 active+clean; 585 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 24 KiB/s wr, 211 op/s; 8/294 objects degraded (2.721%); 24 B/s, 21 keys/s, 4 objects/s recovering 2024-07-28T19:40:21.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:21 smithi138 bash[67894]: cluster 2024-07-28T19:40:20.125415+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-07-28T19:40:22.179 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (986 ms) 2024-07-28T19:40:22.179 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-07-28T19:40:22.430 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:22 smithi100 bash[82644]: cluster 2024-07-28T19:40:21.127261+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-07-28T19:40:22.431 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:22 smithi100 bash[80199]: cluster 2024-07-28T19:40:21.127261+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-07-28T19:40:22.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:22 smithi138 bash[67894]: cluster 2024-07-28T19:40:21.127261+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-07-28T19:40:22.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:40:22 smithi138 bash[60615]: ts=2024-07-28T19:40:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:40:23.136 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:40:22] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2024-07-28T19:40:23.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[80199]: cluster 2024-07-28T19:40:21.650646+0000 mgr.y (mgr.25358) 215 : cluster [DBG] pgmap v388: 105 pgs: 105 active+clean; 586 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 215 KiB/s rd, 28 KiB/s wr, 304 op/s; 58 B/s, 23 keys/s, 6 objects/s recovering 2024-07-28T19:40:23.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[80199]: cluster 2024-07-28T19:40:22.130065+0000 mon.a (mon.0) 677 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/294 objects degraded (2.721%), 4 pgs degraded) 2024-07-28T19:40:23.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[80199]: cluster 2024-07-28T19:40:22.147749+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-07-28T19:40:23.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[80199]: audit 2024-07-28T19:40:22.637544+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:23.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[80199]: audit 2024-07-28T19:40:22.639334+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:23.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[82644]: cluster 2024-07-28T19:40:21.650646+0000 mgr.y (mgr.25358) 215 : cluster [DBG] pgmap v388: 105 pgs: 105 active+clean; 586 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 215 KiB/s rd, 28 KiB/s wr, 304 op/s; 58 B/s, 23 keys/s, 6 objects/s recovering 2024-07-28T19:40:23.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[82644]: cluster 2024-07-28T19:40:22.130065+0000 mon.a (mon.0) 677 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/294 objects degraded (2.721%), 4 pgs degraded) 2024-07-28T19:40:23.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[82644]: cluster 2024-07-28T19:40:22.147749+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-07-28T19:40:23.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[82644]: audit 2024-07-28T19:40:22.637544+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:23.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:23 smithi100 bash[82644]: audit 2024-07-28T19:40:22.639334+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:23 smithi138 bash[67894]: cluster 2024-07-28T19:40:21.650646+0000 mgr.y (mgr.25358) 215 : cluster [DBG] pgmap v388: 105 pgs: 105 active+clean; 586 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 215 KiB/s rd, 28 KiB/s wr, 304 op/s; 58 B/s, 23 keys/s, 6 objects/s recovering 2024-07-28T19:40:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:23 smithi138 bash[67894]: cluster 2024-07-28T19:40:22.130065+0000 mon.a (mon.0) 677 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/294 objects degraded (2.721%), 4 pgs degraded) 2024-07-28T19:40:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:23 smithi138 bash[67894]: cluster 2024-07-28T19:40:22.147749+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-07-28T19:40:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:23 smithi138 bash[67894]: audit 2024-07-28T19:40:22.637544+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:23 smithi138 bash[67894]: audit 2024-07-28T19:40:22.639334+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:24.316 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.ResizeCopyup (2138 ms) 2024-07-28T19:40:24.316 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-07-28T19:40:24.338 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:40:24.338 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.DiscardCopyup (22 ms) 2024-07-28T19:40:24.339 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.ImageOptions 2024-07-28T19:40:24.359 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.ImageOptions (21 ms) 2024-07-28T19:40:24.359 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-07-28T19:40:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[80199]: cluster 2024-07-28T19:40:23.153557+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-07-28T19:40:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[80199]: audit 2024-07-28T19:40:23.180101+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[80199]: audit 2024-07-28T19:40:23.185418+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[80199]: audit 2024-07-28T19:40:23.186799+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[80199]: audit 2024-07-28T19:40:23.188431+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:24.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[80199]: audit 2024-07-28T19:40:23.195776+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[80199]: audit 2024-07-28T19:40:23.299748+0000 mon.a (mon.0) 687 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[82644]: cluster 2024-07-28T19:40:23.153557+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-07-28T19:40:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[82644]: audit 2024-07-28T19:40:23.180101+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[82644]: audit 2024-07-28T19:40:23.185418+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[82644]: audit 2024-07-28T19:40:23.186799+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[82644]: audit 2024-07-28T19:40:23.188431+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[82644]: audit 2024-07-28T19:40:23.195776+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:24 smithi100 bash[82644]: audit 2024-07-28T19:40:23.299748+0000 mon.a (mon.0) 687 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:24 smithi138 bash[67894]: cluster 2024-07-28T19:40:23.153557+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-07-28T19:40:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:24 smithi138 bash[67894]: audit 2024-07-28T19:40:23.180101+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:24 smithi138 bash[67894]: audit 2024-07-28T19:40:23.185418+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:24 smithi138 bash[67894]: audit 2024-07-28T19:40:23.186799+0000 mon.a (mon.0) 684 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:24 smithi138 bash[67894]: audit 2024-07-28T19:40:23.188431+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:24 smithi138 bash[67894]: audit 2024-07-28T19:40:23.195776+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:24 smithi138 bash[67894]: audit 2024-07-28T19:40:23.299748+0000 mon.a (mon.0) 687 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:25.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:25 smithi100 bash[80199]: cluster 2024-07-28T19:40:23.651857+0000 mgr.y (mgr.25358) 216 : cluster [DBG] pgmap v391: 105 pgs: 105 active+clean; 600 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 243 KiB/s rd, 21 KiB/s wr, 324 op/s; 34 B/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:40:25.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:25 smithi100 bash[80199]: cluster 2024-07-28T19:40:24.207007+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-07-28T19:40:25.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:25 smithi100 bash[80199]: cluster 2024-07-28T19:40:24.854723+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-07-28T19:40:25.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:25 smithi100 bash[82644]: cluster 2024-07-28T19:40:23.651857+0000 mgr.y (mgr.25358) 216 : cluster [DBG] pgmap v391: 105 pgs: 105 active+clean; 600 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 243 KiB/s rd, 21 KiB/s wr, 324 op/s; 34 B/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:40:25.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:25 smithi100 bash[82644]: cluster 2024-07-28T19:40:24.207007+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-07-28T19:40:25.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:25 smithi100 bash[82644]: cluster 2024-07-28T19:40:24.854723+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-07-28T19:40:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:25 smithi138 bash[67894]: cluster 2024-07-28T19:40:23.651857+0000 mgr.y (mgr.25358) 216 : cluster [DBG] pgmap v391: 105 pgs: 105 active+clean; 600 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 243 KiB/s rd, 21 KiB/s wr, 324 op/s; 34 B/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:40:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:25 smithi138 bash[67894]: cluster 2024-07-28T19:40:24.207007+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-07-28T19:40:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:25 smithi138 bash[67894]: cluster 2024-07-28T19:40:24.854723+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-07-28T19:40:27.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:26 smithi100 bash[82644]: cluster 2024-07-28T19:40:25.652759+0000 mgr.y (mgr.25358) 217 : cluster [DBG] pgmap v394: 105 pgs: 105 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 23 KiB/s wr, 294 op/s 2024-07-28T19:40:27.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:26 smithi100 bash[82644]: cluster 2024-07-28T19:40:25.855870+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-07-28T19:40:27.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:26 smithi100 bash[80199]: cluster 2024-07-28T19:40:25.652759+0000 mgr.y (mgr.25358) 217 : cluster [DBG] pgmap v394: 105 pgs: 105 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 23 KiB/s wr, 294 op/s 2024-07-28T19:40:27.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:26 smithi100 bash[80199]: cluster 2024-07-28T19:40:25.855870+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-07-28T19:40:27.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:26 smithi138 bash[67894]: cluster 2024-07-28T19:40:25.652759+0000 mgr.y (mgr.25358) 217 : cluster [DBG] pgmap v394: 105 pgs: 105 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 23 KiB/s wr, 294 op/s 2024-07-28T19:40:27.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:26 smithi138 bash[67894]: cluster 2024-07-28T19:40:25.855870+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-07-28T19:40:27.905 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.WriteFullCopyup (3546 ms) 2024-07-28T19:40:27.906 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-07-28T19:40:28.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:27 smithi100 bash[80199]: cluster 2024-07-28T19:40:26.865612+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-07-28T19:40:28.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:27 smithi100 bash[80199]: audit 2024-07-28T19:40:27.743361+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:28.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:27 smithi100 bash[82644]: cluster 2024-07-28T19:40:26.865612+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-07-28T19:40:28.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:27 smithi100 bash[82644]: audit 2024-07-28T19:40:27.743361+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:28.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:27 smithi138 bash[67894]: cluster 2024-07-28T19:40:26.865612+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-07-28T19:40:28.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:27 smithi138 bash[67894]: audit 2024-07-28T19:40:27.743361+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:29.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:28 smithi100 bash[82644]: cluster 2024-07-28T19:40:27.653444+0000 mgr.y (mgr.25358) 218 : cluster [DBG] pgmap v397: 105 pgs: 105 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 5.2 KiB/s wr, 63 op/s 2024-07-28T19:40:29.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:28 smithi100 bash[82644]: cluster 2024-07-28T19:40:27.875771+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-07-28T19:40:29.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:28 smithi100 bash[80199]: cluster 2024-07-28T19:40:27.653444+0000 mgr.y (mgr.25358) 218 : cluster [DBG] pgmap v397: 105 pgs: 105 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 5.2 KiB/s wr, 63 op/s 2024-07-28T19:40:29.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:28 smithi100 bash[80199]: cluster 2024-07-28T19:40:27.875771+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-07-28T19:40:29.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:28 smithi138 bash[67894]: cluster 2024-07-28T19:40:27.653444+0000 mgr.y (mgr.25358) 218 : cluster [DBG] pgmap v397: 105 pgs: 105 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 5.2 KiB/s wr, 63 op/s 2024-07-28T19:40:29.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:28 smithi138 bash[67894]: cluster 2024-07-28T19:40:27.875771+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-07-28T19:40:29.939 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2034 ms) 2024-07-28T19:40:29.939 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.TestCoR 2024-07-28T19:40:29.960 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING due to disabled rbd_copy_on_read 2024-07-28T19:40:29.960 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.TestCoR (20 ms) 2024-07-28T19:40:29.960 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-07-28T19:40:30.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[82644]: cluster 2024-07-28T19:40:28.881962+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-07-28T19:40:30.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[82644]: audit 2024-07-28T19:40:29.151301+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[82644]: audit 2024-07-28T19:40:29.161283+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[82644]: audit 2024-07-28T19:40:29.648167+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:30.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[82644]: audit 2024-07-28T19:40:29.649751+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:30.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[82644]: audit 2024-07-28T19:40:29.658671+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[80199]: cluster 2024-07-28T19:40:28.881962+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-07-28T19:40:30.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[80199]: audit 2024-07-28T19:40:29.151301+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[80199]: audit 2024-07-28T19:40:29.161283+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[80199]: audit 2024-07-28T19:40:29.648167+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:30.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[80199]: audit 2024-07-28T19:40:29.649751+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:30.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:29 smithi100 bash[80199]: audit 2024-07-28T19:40:29.658671+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:29 smithi138 bash[67894]: cluster 2024-07-28T19:40:28.881962+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-07-28T19:40:30.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:29 smithi138 bash[67894]: audit 2024-07-28T19:40:29.151301+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:29 smithi138 bash[67894]: audit 2024-07-28T19:40:29.161283+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:29 smithi138 bash[67894]: audit 2024-07-28T19:40:29.648167+0000 mon.a (mon.0) 697 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:30.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:29 smithi138 bash[67894]: audit 2024-07-28T19:40:29.649751+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:30.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:29 smithi138 bash[67894]: audit 2024-07-28T19:40:29.658671+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout:made parent image "image396": 4194304K (1024 * 4K) 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout:generated random write map: 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout: [1 , 7043 ] 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout: [6 , 25138 ] 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout: [13 , 55450 ] 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout: [25 , 102509 ] 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout: [31 , 129957 ] 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout: [47 , 195558 ] 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout: [54 , 224041 ] 2024-07-28T19:40:30.552 INFO:tasks.workunit.client.0.smithi100.stdout: [85 , 350224 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [101 , 415876 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [108 , 442928 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [114 , 469201 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [123 , 506542 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [164 , 674563 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [165 , 676611 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [180 , 740628 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [217 , 890357 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [219 , 897280 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [221 , 905221 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [230 , 944109 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [245 , 1005820 ] 2024-07-28T19:40:30.553 INFO:tasks.workunit.client.0.smithi100.stdout: [253 , 1038013 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [264 , 1084093 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [280 , 1148811 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [282 , 1157929 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [290 , 1190875 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [298 , 1222284 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [301 , 1234611 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [310 , 1269829 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [326 , 1335601 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [336 , 1379746 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [338 , 1386234 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [349 , 1431805 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [350 , 1436921 ] 2024-07-28T19:40:30.554 INFO:tasks.workunit.client.0.smithi100.stdout: [353 , 1448054 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [359 , 1472125 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [368 , 1507912 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [391 , 1604402 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [403 , 1651963 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [424 , 1739314 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [426 , 1744956 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [427 , 1752320 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [439 , 1801334 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [443 , 1818103 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [451 , 1849065 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [455 , 1864123 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [464 , 1901764 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [468 , 1917532 ] 2024-07-28T19:40:30.555 INFO:tasks.workunit.client.0.smithi100.stdout: [505 , 2071013 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [537 , 2201335 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [545 , 2234547 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [553 , 2268562 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [557 , 2284426 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [560 , 2293798 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [563 , 2306918 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [579 , 2373110 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [592 , 2426256 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [610 , 2499434 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [613 , 2511780 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [614 , 2516187 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [617 , 2529896 ] 2024-07-28T19:40:30.556 INFO:tasks.workunit.client.0.smithi100.stdout: [625 , 2560869 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [650 , 2662583 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [664 , 2721777 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [670 , 2744769 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [675 , 2767695 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [683 , 2799655 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [712 , 2917747 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [734 , 3006802 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [749 , 3068438 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [763 , 3125909 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [768 , 3149278 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [770 , 3157037 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [781 , 3199354 ] 2024-07-28T19:40:30.557 INFO:tasks.workunit.client.0.smithi100.stdout: [803 , 3289831 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [810 , 3318699 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [811 , 3324621 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [832 , 3410029 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [838 , 3432641 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [851 , 3488165 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [862 , 3532743 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [866 , 3548577 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [870 , 3565790 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [872 , 3574074 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [889 , 3642134 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [902 , 3697477 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [906 , 3714384 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [913 , 3740708 ] 2024-07-28T19:40:30.558 INFO:tasks.workunit.client.0.smithi100.stdout: [924 , 3786975 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: [950 , 3893030 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: [958 , 3926673 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: [962 , 3943470 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: [972 , 3981956 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: [993 , 4068042 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: [999 , 4093772 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: [1012 , 4146488 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: [1014 , 4156302 ] 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout:write data based on random map 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: write object-1 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: write object-6 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: write object-13 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: write object-25 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: write object-31 2024-07-28T19:40:30.559 INFO:tasks.workunit.client.0.smithi100.stdout: write object-47 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-54 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-85 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-101 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-108 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-114 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-123 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-164 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-165 2024-07-28T19:40:30.560 INFO:tasks.workunit.client.0.smithi100.stdout: write object-180 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-217 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-219 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-221 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-230 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-245 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-253 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-264 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-280 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-282 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-290 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-298 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-301 2024-07-28T19:40:30.561 INFO:tasks.workunit.client.0.smithi100.stdout: write object-310 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-326 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-336 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-338 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-349 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-350 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-353 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-359 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-368 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-391 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-403 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-424 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-426 2024-07-28T19:40:30.562 INFO:tasks.workunit.client.0.smithi100.stdout: write object-427 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-439 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-443 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-451 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-455 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-464 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-468 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-505 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-537 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-545 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-553 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-557 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-560 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-563 2024-07-28T19:40:30.563 INFO:tasks.workunit.client.0.smithi100.stdout: write object-579 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-592 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-610 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-613 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-614 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-617 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-625 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-650 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-664 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-670 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-675 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-683 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-712 2024-07-28T19:40:30.564 INFO:tasks.workunit.client.0.smithi100.stdout: write object-734 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-749 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-763 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-768 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-770 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-781 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-803 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-810 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-811 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-832 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-838 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-851 2024-07-28T19:40:30.565 INFO:tasks.workunit.client.0.smithi100.stdout: write object-862 2024-07-28T19:40:30.566 INFO:tasks.workunit.client.0.smithi100.stdout: write object-866 2024-07-28T19:40:30.566 INFO:tasks.workunit.client.0.smithi100.stdout: write object-870 2024-07-28T19:40:30.566 INFO:tasks.workunit.client.0.smithi100.stdout: write object-872 2024-07-28T19:40:30.566 INFO:tasks.workunit.client.0.smithi100.stdout: write object-889 2024-07-28T19:40:30.566 INFO:tasks.workunit.client.0.smithi100.stdout: write object-902 2024-07-28T19:40:30.566 INFO:tasks.workunit.client.0.smithi100.stdout: write object-906 2024-07-28T19:40:30.566 INFO:tasks.workunit.client.0.smithi100.stdout: write object-913 2024-07-28T19:40:30.566 INFO:tasks.workunit.client.0.smithi100.stdout: write object-924 2024-07-28T19:40:31.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:30 smithi100 bash[82644]: cluster 2024-07-28T19:40:29.654208+0000 mgr.y (mgr.25358) 219 : cluster [DBG] pgmap v400: 105 pgs: 105 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 255 op/s 2024-07-28T19:40:31.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:30 smithi100 bash[82644]: cluster 2024-07-28T19:40:29.891608+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-07-28T19:40:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:30 smithi100 bash[80199]: cluster 2024-07-28T19:40:29.654208+0000 mgr.y (mgr.25358) 219 : cluster [DBG] pgmap v400: 105 pgs: 105 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 255 op/s 2024-07-28T19:40:31.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:30 smithi100 bash[80199]: cluster 2024-07-28T19:40:29.891608+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-07-28T19:40:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:30 smithi138 bash[67894]: cluster 2024-07-28T19:40:29.654208+0000 mgr.y (mgr.25358) 219 : cluster [DBG] pgmap v400: 105 pgs: 105 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 255 op/s 2024-07-28T19:40:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:30 smithi138 bash[67894]: cluster 2024-07-28T19:40:29.891608+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-07-28T19:40:32.003 INFO:tasks.workunit.client.0.smithi100.stdout: write object-950 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout: write object-958 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout: write object-962 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout: write object-972 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout: write object-993 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout: write object-999 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout: write object-1012 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout: write object-1014 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout:verify written data by reading 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout: read object-1 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout:made snapshot "image396@parent_snap" and protect it 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout:made and opened clone "image397" 2024-07-28T19:40:32.004 INFO:tasks.workunit.client.0.smithi100.stdout:flattening clone: "image397" 2024-07-28T19:40:32.005 INFO:tasks.workunit.client.0.smithi100.stdout:check whether child image has the same set of objects as parent 2024-07-28T19:40:32.005 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (2044 ms) 2024-07-28T19:40:32.005 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-07-28T19:40:32.109 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.PoolMetadataConfApply (106 ms) 2024-07-28T19:40:32.109 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.Sparsify 2024-07-28T19:40:32.137 INFO:tasks.workunit.client.0.smithi100.stdout:sparsify_supported=1 2024-07-28T19:40:32.137 INFO:tasks.workunit.client.0.smithi100.stdout:sparse_read_supported=1 2024-07-28T19:40:32.179 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.Sparsify (70 ms) 2024-07-28T19:40:32.179 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.SparsifyClone 2024-07-28T19:40:32.214 INFO:tasks.workunit.client.0.smithi100.stdout:sparsify_supported=1 2024-07-28T19:40:32.378 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[73305]: [28/Jul/2024:19:40:32] ENGINE Bus STOPPING 2024-07-28T19:40:32.657 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[73305]: [28/Jul/2024:19:40:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:40:32.658 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[73305]: [28/Jul/2024:19:40:32] ENGINE Bus STOPPED 2024-07-28T19:40:32.658 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[73305]: [28/Jul/2024:19:40:32] ENGINE Bus STARTING 2024-07-28T19:40:32.658 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[73305]: [28/Jul/2024:19:40:32] ENGINE Serving on http://:::9283 2024-07-28T19:40:32.658 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[73305]: [28/Jul/2024:19:40:32] ENGINE Bus STARTED 2024-07-28T19:40:32.659 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:40:32 smithi138 bash[60615]: ts=2024-07-28T19:40:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:40:32.996 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:40:32] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2024-07-28T19:40:32.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[82644]: cluster 2024-07-28T19:40:31.655032+0000 mgr.y (mgr.25358) 220 : cluster [DBG] pgmap v402: 105 pgs: 105 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 214 op/s 2024-07-28T19:40:32.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[82644]: cluster 2024-07-28T19:40:31.665081+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-07-28T19:40:32.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[82644]: audit 2024-07-28T19:40:32.036390+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' 2024-07-28T19:40:32.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[82644]: audit 2024-07-28T19:40:32.051554+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[82644]: audit 2024-07-28T19:40:32.053634+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[82644]: audit 2024-07-28T19:40:32.054554+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[82644]: audit 2024-07-28T19:40:32.063548+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[80199]: cluster 2024-07-28T19:40:31.655032+0000 mgr.y (mgr.25358) 220 : cluster [DBG] pgmap v402: 105 pgs: 105 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 214 op/s 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[80199]: cluster 2024-07-28T19:40:31.665081+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[80199]: audit 2024-07-28T19:40:32.036390+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[80199]: audit 2024-07-28T19:40:32.051554+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[80199]: audit 2024-07-28T19:40:32.053634+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[80199]: audit 2024-07-28T19:40:32.054554+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:32 smithi100 bash[80199]: audit 2024-07-28T19:40:32.063548+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:32 smithi138 bash[67894]: cluster 2024-07-28T19:40:31.655032+0000 mgr.y (mgr.25358) 220 : cluster [DBG] pgmap v402: 105 pgs: 105 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.5 MiB/s wr, 214 op/s 2024-07-28T19:40:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:32 smithi138 bash[67894]: cluster 2024-07-28T19:40:31.665081+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-07-28T19:40:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:32 smithi138 bash[67894]: audit 2024-07-28T19:40:32.036390+0000 mon.a (mon.0) 702 : audit [INF] from='client.? ' entity='client.admin' 2024-07-28T19:40:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:32 smithi138 bash[67894]: audit 2024-07-28T19:40:32.051554+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:40:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:32 smithi138 bash[67894]: audit 2024-07-28T19:40:32.053634+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:40:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:32 smithi138 bash[67894]: audit 2024-07-28T19:40:32.054554+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:40:33.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:32 smithi138 bash[67894]: audit 2024-07-28T19:40:32.063548+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:40:34.104 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.SparsifyClone (1924 ms) 2024-07-28T19:40:34.104 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestInternal.MissingDataPool 2024-07-28T19:40:34.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:34 smithi100 bash[80199]: cluster 2024-07-28T19:40:33.068342+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-07-28T19:40:34.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:34 smithi100 bash[82644]: cluster 2024-07-28T19:40:33.068342+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-07-28T19:40:34.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:34 smithi138 bash[67894]: cluster 2024-07-28T19:40:33.068342+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-07-28T19:40:35.131 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestInternal.MissingDataPool (1028 ms) 2024-07-28T19:40:35.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:35 smithi100 bash[80199]: cluster 2024-07-28T19:40:33.656162+0000 mgr.y (mgr.25358) 221 : cluster [DBG] pgmap v405: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 144 KiB/s wr, 682 op/s 2024-07-28T19:40:35.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:35 smithi100 bash[80199]: cluster 2024-07-28T19:40:34.084229+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-07-28T19:40:35.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:35 smithi100 bash[80199]: cluster 2024-07-28T19:40:34.856184+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-07-28T19:40:35.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:35 smithi100 bash[82644]: cluster 2024-07-28T19:40:33.656162+0000 mgr.y (mgr.25358) 221 : cluster [DBG] pgmap v405: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 144 KiB/s wr, 682 op/s 2024-07-28T19:40:35.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:35 smithi100 bash[82644]: cluster 2024-07-28T19:40:34.084229+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-07-28T19:40:35.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:35 smithi100 bash[82644]: cluster 2024-07-28T19:40:34.856184+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-07-28T19:40:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:35 smithi138 bash[67894]: cluster 2024-07-28T19:40:33.656162+0000 mgr.y (mgr.25358) 221 : cluster [DBG] pgmap v405: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 441 KiB/s rd, 144 KiB/s wr, 682 op/s 2024-07-28T19:40:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:35 smithi138 bash[67894]: cluster 2024-07-28T19:40:34.084229+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-07-28T19:40:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:35 smithi138 bash[67894]: cluster 2024-07-28T19:40:34.856184+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-07-28T19:40:36.100 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 34 tests from TestInternal (38249 ms total) 2024-07-28T19:40:36.100 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:40:36.100 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 17 tests from TestImageWatcher 2024-07-28T19:40:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:37 smithi100 bash[80199]: cluster 2024-07-28T19:40:35.656987+0000 mgr.y (mgr.25358) 222 : cluster [DBG] pgmap v408: 105 pgs: 105 active+clean; 5.1 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 580 KiB/s rd, 190 KiB/s wr, 913 op/s 2024-07-28T19:40:37.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:37 smithi100 bash[80199]: cluster 2024-07-28T19:40:36.096230+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-07-28T19:40:37.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:37 smithi100 bash[82644]: cluster 2024-07-28T19:40:35.656987+0000 mgr.y (mgr.25358) 222 : cluster [DBG] pgmap v408: 105 pgs: 105 active+clean; 5.1 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 580 KiB/s rd, 190 KiB/s wr, 913 op/s 2024-07-28T19:40:37.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:37 smithi100 bash[82644]: cluster 2024-07-28T19:40:36.096230+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-07-28T19:40:37.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:37 smithi138 bash[67894]: cluster 2024-07-28T19:40:35.656987+0000 mgr.y (mgr.25358) 222 : cluster [DBG] pgmap v408: 105 pgs: 105 active+clean; 5.1 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 580 KiB/s rd, 190 KiB/s wr, 913 op/s 2024-07-28T19:40:37.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:37 smithi138 bash[67894]: cluster 2024-07-28T19:40:36.096230+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-07-28T19:40:38.100 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-07-28T19:40:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[80199]: cluster 2024-07-28T19:40:37.097452+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-07-28T19:40:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[80199]: audit 2024-07-28T19:40:37.100514+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.100:0/3592310693' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:38.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[80199]: audit 2024-07-28T19:40:37.102274+0000 mon.a (mon.0) 712 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[80199]: audit 2024-07-28T19:40:37.632231+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[80199]: audit 2024-07-28T19:40:38.102354+0000 mon.a (mon.0) 714 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[80199]: cluster 2024-07-28T19:40:38.102966+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[82644]: cluster 2024-07-28T19:40:37.097452+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[82644]: audit 2024-07-28T19:40:37.100514+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.100:0/3592310693' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[82644]: audit 2024-07-28T19:40:37.102274+0000 mon.a (mon.0) 712 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[82644]: audit 2024-07-28T19:40:37.632231+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[82644]: audit 2024-07-28T19:40:38.102354+0000 mon.a (mon.0) 714 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:38.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:38 smithi100 bash[82644]: cluster 2024-07-28T19:40:38.102966+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-07-28T19:40:38.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:38 smithi138 bash[67894]: cluster 2024-07-28T19:40:37.097452+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-07-28T19:40:38.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:38 smithi138 bash[67894]: audit 2024-07-28T19:40:37.100514+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.100:0/3592310693' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:38 smithi138 bash[67894]: audit 2024-07-28T19:40:37.102274+0000 mon.a (mon.0) 712 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:38 smithi138 bash[67894]: audit 2024-07-28T19:40:37.632231+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:38 smithi138 bash[67894]: audit 2024-07-28T19:40:38.102354+0000 mon.a (mon.0) 714 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:38 smithi138 bash[67894]: cluster 2024-07-28T19:40:38.102966+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-07-28T19:40:39.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:39 smithi100 bash[80199]: cluster 2024-07-28T19:40:37.657642+0000 mgr.y (mgr.25358) 223 : cluster [DBG] pgmap v411: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:39.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:39 smithi100 bash[80199]: cluster 2024-07-28T19:40:38.112620+0000 mon.a (mon.0) 716 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:40:39.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:39 smithi100 bash[80199]: cluster 2024-07-28T19:40:39.100651+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-07-28T19:40:39.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:39 smithi100 bash[82644]: cluster 2024-07-28T19:40:37.657642+0000 mgr.y (mgr.25358) 223 : cluster [DBG] pgmap v411: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:39.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:39 smithi100 bash[82644]: cluster 2024-07-28T19:40:38.112620+0000 mon.a (mon.0) 716 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:40:39.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:39 smithi100 bash[82644]: cluster 2024-07-28T19:40:39.100651+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-07-28T19:40:39.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:39 smithi138 bash[67894]: cluster 2024-07-28T19:40:37.657642+0000 mgr.y (mgr.25358) 223 : cluster [DBG] pgmap v411: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:39.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:39 smithi138 bash[67894]: cluster 2024-07-28T19:40:38.112620+0000 mon.a (mon.0) 716 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:40:39.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:39 smithi138 bash[67894]: cluster 2024-07-28T19:40:39.100651+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-07-28T19:40:40.168 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (2068 ms) 2024-07-28T19:40:40.168 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-07-28T19:40:40.213 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyFlatten (44 ms) 2024-07-28T19:40:40.213 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-07-28T19:40:40.263 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyResize (51 ms) 2024-07-28T19:40:40.263 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-07-28T19:40:40.302 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (38 ms) 2024-07-28T19:40:40.302 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-07-28T19:40:40.334 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (33 ms) 2024-07-28T19:40:40.334 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-07-28T19:40:40.375 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (41 ms) 2024-07-28T19:40:40.376 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-07-28T19:40:40.420 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (45 ms) 2024-07-28T19:40:40.420 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-07-28T19:40:40.478 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifySnapRename (58 ms) 2024-07-28T19:40:40.478 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-07-28T19:40:40.529 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (50 ms) 2024-07-28T19:40:40.529 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-07-28T19:40:40.585 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (57 ms) 2024-07-28T19:40:40.585 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-07-28T19:40:40.637 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (52 ms) 2024-07-28T19:40:40.637 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-07-28T19:40:40.691 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (54 ms) 2024-07-28T19:40:40.691 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-07-28T19:40:40.746 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyRename (54 ms) 2024-07-28T19:40:40.746 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-07-28T19:40:40.794 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (49 ms) 2024-07-28T19:40:40.794 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-07-28T19:40:40.831 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (37 ms) 2024-07-28T19:40:40.831 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-07-28T19:40:40.879 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (48 ms) 2024-07-28T19:40:40.880 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-07-28T19:40:40.930 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (51 ms) 2024-07-28T19:40:41.149 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 17 tests from TestImageWatcher (2830 ms total) 2024-07-28T19:40:41.149 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:40:41.149 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 160 tests from TestLibRBD 2024-07-28T19:40:41.191 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-07-28T19:40:41.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:41 smithi100 bash[80199]: cluster 2024-07-28T19:40:39.658300+0000 mgr.y (mgr.25358) 224 : cluster [DBG] pgmap v414: 105 pgs: 6 unknown, 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:41.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:41 smithi100 bash[80199]: cluster 2024-07-28T19:40:40.104207+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e564: 8 total, 8 up, 8 in 2024-07-28T19:40:41.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:41 smithi100 bash[80199]: cluster 2024-07-28T19:40:40.135552+0000 mon.a (mon.0) 719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:40:41.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:41 smithi100 bash[82644]: cluster 2024-07-28T19:40:39.658300+0000 mgr.y (mgr.25358) 224 : cluster [DBG] pgmap v414: 105 pgs: 6 unknown, 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:41.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:41 smithi100 bash[82644]: cluster 2024-07-28T19:40:40.104207+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e564: 8 total, 8 up, 8 in 2024-07-28T19:40:41.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:41 smithi100 bash[82644]: cluster 2024-07-28T19:40:40.135552+0000 mon.a (mon.0) 719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:40:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:41 smithi138 bash[67894]: cluster 2024-07-28T19:40:39.658300+0000 mgr.y (mgr.25358) 224 : cluster [DBG] pgmap v414: 105 pgs: 6 unknown, 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:41 smithi138 bash[67894]: cluster 2024-07-28T19:40:40.104207+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e564: 8 total, 8 up, 8 in 2024-07-28T19:40:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:41 smithi138 bash[67894]: cluster 2024-07-28T19:40:40.135552+0000 mon.a (mon.0) 719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:40:42.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:42 smithi100 bash[80199]: cluster 2024-07-28T19:40:41.147253+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e565: 8 total, 8 up, 8 in 2024-07-28T19:40:42.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:42 smithi100 bash[82644]: cluster 2024-07-28T19:40:41.147253+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e565: 8 total, 8 up, 8 in 2024-07-28T19:40:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:42 smithi138 bash[67894]: cluster 2024-07-28T19:40:41.147253+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e565: 8 total, 8 up, 8 in 2024-07-28T19:40:42.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:40:42 smithi138 bash[60615]: ts=2024-07-28T19:40:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:40:43.170 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:40:42] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2024-07-28T19:40:43.176 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:43.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:43 smithi100 bash[80199]: cluster 2024-07-28T19:40:41.658935+0000 mgr.y (mgr.25358) 225 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:43.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:43 smithi100 bash[80199]: cluster 2024-07-28T19:40:42.161467+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e566: 8 total, 8 up, 8 in 2024-07-28T19:40:43.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:43 smithi100 bash[80199]: audit 2024-07-28T19:40:42.164691+0000 mon.a (mon.0) 722 : audit [INF] from='client.? 172.21.15.100:0/2392463147' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:43.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:43 smithi100 bash[82644]: cluster 2024-07-28T19:40:41.658935+0000 mgr.y (mgr.25358) 225 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:43 smithi100 bash[82644]: cluster 2024-07-28T19:40:42.161467+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e566: 8 total, 8 up, 8 in 2024-07-28T19:40:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:43 smithi100 bash[82644]: audit 2024-07-28T19:40:42.164691+0000 mon.a (mon.0) 722 : audit [INF] from='client.? 172.21.15.100:0/2392463147' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:43 smithi138 bash[67894]: cluster 2024-07-28T19:40:41.658935+0000 mgr.y (mgr.25358) 225 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:43 smithi138 bash[67894]: cluster 2024-07-28T19:40:42.161467+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e566: 8 total, 8 up, 8 in 2024-07-28T19:40:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:43 smithi138 bash[67894]: audit 2024-07-28T19:40:42.164691+0000 mon.a (mon.0) 722 : audit [INF] from='client.? 172.21.15.100:0/2392463147' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:44.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:44 smithi100 bash[82644]: audit 2024-07-28T19:40:43.162136+0000 mon.a (mon.0) 723 : audit [INF] from='client.? 172.21.15.100:0/2392463147' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:44.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:44 smithi100 bash[82644]: cluster 2024-07-28T19:40:43.162673+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e567: 8 total, 8 up, 8 in 2024-07-28T19:40:44.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:44 smithi100 bash[80199]: audit 2024-07-28T19:40:43.162136+0000 mon.a (mon.0) 723 : audit [INF] from='client.? 172.21.15.100:0/2392463147' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:44.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:44 smithi100 bash[80199]: cluster 2024-07-28T19:40:43.162673+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e567: 8 total, 8 up, 8 in 2024-07-28T19:40:44.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:44 smithi138 bash[67894]: audit 2024-07-28T19:40:43.162136+0000 mon.a (mon.0) 723 : audit [INF] from='client.? 172.21.15.100:0/2392463147' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:44.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:44 smithi138 bash[67894]: cluster 2024-07-28T19:40:43.162673+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e567: 8 total, 8 up, 8 in 2024-07-28T19:40:45.257 INFO:tasks.workunit.client.0.smithi100.stdout:image has size 2097152 and order 22 2024-07-28T19:40:45.257 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.CreateAndStat (4065 ms) 2024-07-28T19:40:45.257 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-07-28T19:40:45.257 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.293 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (37 ms) 2024-07-28T19:40:45.293 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-07-28T19:40:45.294 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.334 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.CreateAndStatPP (41 ms) 2024-07-28T19:40:45.335 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.GetId 2024-07-28T19:40:45.335 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.376 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.GetId (41 ms) 2024-07-28T19:40:45.376 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-07-28T19:40:45.376 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.431 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.GetIdPP (56 ms) 2024-07-28T19:40:45.431 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-07-28T19:40:45.431 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.467 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (36 ms) 2024-07-28T19:40:45.468 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-07-28T19:40:45.468 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:45 smithi100 bash[82644]: cluster 2024-07-28T19:40:43.659969+0000 mgr.y (mgr.25358) 226 : cluster [DBG] pgmap v420: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:45.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:45 smithi100 bash[82644]: cluster 2024-07-28T19:40:44.195798+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e568: 8 total, 8 up, 8 in 2024-07-28T19:40:45.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:45 smithi100 bash[80199]: cluster 2024-07-28T19:40:43.659969+0000 mgr.y (mgr.25358) 226 : cluster [DBG] pgmap v420: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:45.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:45 smithi100 bash[80199]: cluster 2024-07-28T19:40:44.195798+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e568: 8 total, 8 up, 8 in 2024-07-28T19:40:45.509 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (41 ms) 2024-07-28T19:40:45.509 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-07-28T19:40:45.509 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.554 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (45 ms) 2024-07-28T19:40:45.554 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-07-28T19:40:45.554 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:45 smithi138 bash[67894]: cluster 2024-07-28T19:40:43.659969+0000 mgr.y (mgr.25358) 226 : cluster [DBG] pgmap v420: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:45 smithi138 bash[67894]: cluster 2024-07-28T19:40:44.195798+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e568: 8 total, 8 up, 8 in 2024-07-28T19:40:45.595 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (42 ms) 2024-07-28T19:40:45.595 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.OpenAio 2024-07-28T19:40:45.595 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.642 INFO:tasks.workunit.client.0.smithi100.stdout:image has size 2097152 and order 22 2024-07-28T19:40:45.642 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.OpenAio (46 ms) 2024-07-28T19:40:45.642 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-07-28T19:40:45.649 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.OpenAioFail (8 ms) 2024-07-28T19:40:45.649 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-07-28T19:40:45.649 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.710 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.OpenAioPP (61 ms) 2024-07-28T19:40:45.710 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-07-28T19:40:45.717 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.OpenAioFailPP (7 ms) 2024-07-28T19:40:45.717 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-07-28T19:40:45.717 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.817 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ResizeAndStat (100 ms) 2024-07-28T19:40:45.818 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-07-28T19:40:45.818 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.896 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (79 ms) 2024-07-28T19:40:45.896 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-07-28T19:40:45.896 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:45.978 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (81 ms) 2024-07-28T19:40:45.978 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-07-28T19:40:45.978 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:46.059 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (82 ms) 2024-07-28T19:40:46.060 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-07-28T19:40:46.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:46 smithi138 bash[67894]: cluster 2024-07-28T19:40:45.206132+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e569: 8 total, 8 up, 8 in 2024-07-28T19:40:46.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:46 smithi100 bash[80199]: cluster 2024-07-28T19:40:45.206132+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e569: 8 total, 8 up, 8 in 2024-07-28T19:40:46.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:46 smithi100 bash[82644]: cluster 2024-07-28T19:40:45.206132+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e569: 8 total, 8 up, 8 in 2024-07-28T19:40:47.629 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:47 smithi138 bash[67894]: cluster 2024-07-28T19:40:45.660615+0000 mgr.y (mgr.25358) 227 : cluster [DBG] pgmap v423: 105 pgs: 105 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:47 smithi138 bash[67894]: cluster 2024-07-28T19:40:46.592879+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e570: 8 total, 8 up, 8 in 2024-07-28T19:40:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:47 smithi138 bash[67894]: audit 2024-07-28T19:40:46.595435+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.100:0/1038770986' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:47 smithi138 bash[67894]: audit 2024-07-28T19:40:46.596229+0000 mon.a (mon.0) 728 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:47.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:47 smithi100 bash[80199]: cluster 2024-07-28T19:40:45.660615+0000 mgr.y (mgr.25358) 227 : cluster [DBG] pgmap v423: 105 pgs: 105 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:47.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:47 smithi100 bash[80199]: cluster 2024-07-28T19:40:46.592879+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e570: 8 total, 8 up, 8 in 2024-07-28T19:40:47.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:47 smithi100 bash[80199]: audit 2024-07-28T19:40:46.595435+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.100:0/1038770986' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:47.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:47 smithi100 bash[80199]: audit 2024-07-28T19:40:46.596229+0000 mon.a (mon.0) 728 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:47.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:47 smithi100 bash[82644]: cluster 2024-07-28T19:40:45.660615+0000 mgr.y (mgr.25358) 227 : cluster [DBG] pgmap v423: 105 pgs: 105 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:47.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:47 smithi100 bash[82644]: cluster 2024-07-28T19:40:46.592879+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e570: 8 total, 8 up, 8 in 2024-07-28T19:40:47.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:47 smithi100 bash[82644]: audit 2024-07-28T19:40:46.595435+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.100:0/1038770986' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:47.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:47 smithi100 bash[82644]: audit 2024-07-28T19:40:46.596229+0000 mon.a (mon.0) 728 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:48.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:48 smithi100 bash[80199]: audit 2024-07-28T19:40:47.598701+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:48.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:48 smithi100 bash[80199]: cluster 2024-07-28T19:40:47.599223+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2024-07-28T19:40:48.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:48 smithi100 bash[80199]: cluster 2024-07-28T19:40:47.661209+0000 mgr.y (mgr.25358) 228 : cluster [DBG] pgmap v426: 137 pgs: 32 unknown, 105 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:48.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:48 smithi100 bash[82644]: audit 2024-07-28T19:40:47.598701+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:48.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:48 smithi100 bash[82644]: cluster 2024-07-28T19:40:47.599223+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2024-07-28T19:40:48.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:48 smithi100 bash[82644]: cluster 2024-07-28T19:40:47.661209+0000 mgr.y (mgr.25358) 228 : cluster [DBG] pgmap v426: 137 pgs: 32 unknown, 105 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:49.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:48 smithi138 bash[67894]: audit 2024-07-28T19:40:47.598701+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:49.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:48 smithi138 bash[67894]: cluster 2024-07-28T19:40:47.599223+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2024-07-28T19:40:49.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:48 smithi138 bash[67894]: cluster 2024-07-28T19:40:47.661209+0000 mgr.y (mgr.25358) 228 : cluster [DBG] pgmap v426: 137 pgs: 32 unknown, 105 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:40:49.650 INFO:tasks.workunit.client.0.smithi100.stdout:image: image18 2024-07-28T19:40:49.650 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image18 2024-07-28T19:40:49.650 INFO:tasks.workunit.client.0.smithi100.stdout:found image18 2024-07-28T19:40:49.650 INFO:tasks.workunit.client.0.smithi100.stdout:erased image18 2024-07-28T19:40:49.650 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:49.848 INFO:tasks.workunit.client.0.smithi100.stdout:image: image18 2024-07-28T19:40:49.848 INFO:tasks.workunit.client.0.smithi100.stdout:image: image19 2024-07-28T19:40:49.848 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image18 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:found image18 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:erased image18 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image19 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:found image19 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:erased image19 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:image: image19 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image19 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:found image19 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:erased image19 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (3789 ms) 2024-07-28T19:40:49.849 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-07-28T19:40:49.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:49 smithi100 bash[80199]: cluster 2024-07-28T19:40:48.613489+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2024-07-28T19:40:49.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:49 smithi100 bash[82644]: cluster 2024-07-28T19:40:48.613489+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2024-07-28T19:40:50.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:49 smithi138 bash[67894]: cluster 2024-07-28T19:40:48.613489+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2024-07-28T19:40:50.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:50 smithi100 bash[80199]: cluster 2024-07-28T19:40:49.614975+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2024-07-28T19:40:50.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:50 smithi100 bash[80199]: cluster 2024-07-28T19:40:49.661936+0000 mgr.y (mgr.25358) 229 : cluster [DBG] pgmap v429: 137 pgs: 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 18 KiB/s wr, 227 op/s 2024-07-28T19:40:50.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:50 smithi100 bash[82644]: cluster 2024-07-28T19:40:49.614975+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2024-07-28T19:40:50.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:50 smithi100 bash[82644]: cluster 2024-07-28T19:40:49.661936+0000 mgr.y (mgr.25358) 229 : cluster [DBG] pgmap v429: 137 pgs: 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 18 KiB/s wr, 227 op/s 2024-07-28T19:40:51.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:50 smithi138 bash[67894]: cluster 2024-07-28T19:40:49.614975+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2024-07-28T19:40:51.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:50 smithi138 bash[67894]: cluster 2024-07-28T19:40:49.661936+0000 mgr.y (mgr.25358) 229 : cluster [DBG] pgmap v429: 137 pgs: 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 18 KiB/s wr, 227 op/s 2024-07-28T19:40:51.649 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:51.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:51 smithi100 bash[80199]: cluster 2024-07-28T19:40:50.634382+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2024-07-28T19:40:51.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:51 smithi100 bash[80199]: audit 2024-07-28T19:40:50.637891+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.100:0/81493505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:51.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:51 smithi100 bash[80199]: audit 2024-07-28T19:40:50.639034+0000 mon.a (mon.0) 734 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:51.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:51 smithi100 bash[82644]: cluster 2024-07-28T19:40:50.634382+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2024-07-28T19:40:51.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:51 smithi100 bash[82644]: audit 2024-07-28T19:40:50.637891+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.100:0/81493505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:51.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:51 smithi100 bash[82644]: audit 2024-07-28T19:40:50.639034+0000 mon.a (mon.0) 734 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:52.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:51 smithi138 bash[67894]: cluster 2024-07-28T19:40:50.634382+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2024-07-28T19:40:52.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:51 smithi138 bash[67894]: audit 2024-07-28T19:40:50.637891+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.100:0/81493505' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:52.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:51 smithi138 bash[67894]: audit 2024-07-28T19:40:50.639034+0000 mon.a (mon.0) 734 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:52.653 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:40:52 smithi138 bash[60615]: ts=2024-07-28T19:40:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:40:52.996 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:40:52] "GET /metrics HTTP/1.1" 200 41274 "" "Prometheus/2.43.0" 2024-07-28T19:40:52.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[82644]: audit 2024-07-28T19:40:51.636643+0000 mon.a (mon.0) 735 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:52.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[82644]: cluster 2024-07-28T19:40:51.637124+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-07-28T19:40:52.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[82644]: cluster 2024-07-28T19:40:51.662552+0000 mgr.y (mgr.25358) 230 : cluster [DBG] pgmap v432: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-07-28T19:40:52.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[82644]: audit 2024-07-28T19:40:52.632563+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:52.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[80199]: audit 2024-07-28T19:40:51.636643+0000 mon.a (mon.0) 735 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:52.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[80199]: cluster 2024-07-28T19:40:51.637124+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-07-28T19:40:52.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[80199]: cluster 2024-07-28T19:40:51.662552+0000 mgr.y (mgr.25358) 230 : cluster [DBG] pgmap v432: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-07-28T19:40:52.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:52 smithi100 bash[80199]: audit 2024-07-28T19:40:52.632563+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:52 smithi138 bash[67894]: audit 2024-07-28T19:40:51.636643+0000 mon.a (mon.0) 735 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:52 smithi138 bash[67894]: cluster 2024-07-28T19:40:51.637124+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-07-28T19:40:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:52 smithi138 bash[67894]: cluster 2024-07-28T19:40:51.662552+0000 mgr.y (mgr.25358) 230 : cluster [DBG] pgmap v432: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-07-28T19:40:53.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:52 smithi138 bash[67894]: audit 2024-07-28T19:40:52.632563+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:40:53.704 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 1 2024-07-28T19:40:53.704 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:40:53.704 INFO:tasks.workunit.client.0.smithi100.stdout:image: image20 2024-07-28T19:40:53.704 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image20 2024-07-28T19:40:53.704 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:53.733 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 2 2024-07-28T19:40:53.733 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:40:53.733 INFO:tasks.workunit.client.0.smithi100.stdout:image: image20 2024-07-28T19:40:53.733 INFO:tasks.workunit.client.0.smithi100.stdout:image: image21 2024-07-28T19:40:53.734 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image20 2024-07-28T19:40:53.734 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image21 2024-07-28T19:40:53.878 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 1 2024-07-28T19:40:53.879 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:40:53.879 INFO:tasks.workunit.client.0.smithi100.stdout:image: image21 2024-07-28T19:40:53.879 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image21 2024-07-28T19:40:53.879 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (4030 ms) 2024-07-28T19:40:53.879 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCopy 2024-07-28T19:40:53.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:53 smithi100 bash[82644]: cluster 2024-07-28T19:40:52.663844+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-07-28T19:40:53.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:53 smithi100 bash[80199]: cluster 2024-07-28T19:40:52.663844+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-07-28T19:40:54.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:53 smithi138 bash[67894]: cluster 2024-07-28T19:40:52.663844+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-07-28T19:40:54.870 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:40:54.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:54 smithi100 bash[80199]: cluster 2024-07-28T19:40:53.664443+0000 mgr.y (mgr.25358) 231 : cluster [DBG] pgmap v434: 169 pgs: 7 unknown, 162 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 5.4 KiB/s wr, 63 op/s 2024-07-28T19:40:54.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:54 smithi100 bash[80199]: cluster 2024-07-28T19:40:53.674205+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-07-28T19:40:54.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:54 smithi100 bash[82644]: cluster 2024-07-28T19:40:53.664443+0000 mgr.y (mgr.25358) 231 : cluster [DBG] pgmap v434: 169 pgs: 7 unknown, 162 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 5.4 KiB/s wr, 63 op/s 2024-07-28T19:40:54.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:54 smithi100 bash[82644]: cluster 2024-07-28T19:40:53.674205+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-07-28T19:40:55.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:54 smithi138 bash[67894]: cluster 2024-07-28T19:40:53.664443+0000 mgr.y (mgr.25358) 231 : cluster [DBG] pgmap v434: 169 pgs: 7 unknown, 162 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 5.4 KiB/s wr, 63 op/s 2024-07-28T19:40:55.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:54 smithi138 bash[67894]: cluster 2024-07-28T19:40:53.674205+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-07-28T19:40:55.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:55 smithi100 bash[80199]: cluster 2024-07-28T19:40:54.681346+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-07-28T19:40:55.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:55 smithi100 bash[80199]: audit 2024-07-28T19:40:54.682902+0000 mon.a (mon.0) 741 : audit [INF] from='client.? 172.21.15.100:0/801420757' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:55.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:55 smithi100 bash[80199]: audit 2024-07-28T19:40:54.868768+0000 mon.a (mon.0) 742 : audit [INF] from='client.? 172.21.15.100:0/801420757' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:55.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:55 smithi100 bash[80199]: cluster 2024-07-28T19:40:54.869338+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-07-28T19:40:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:55 smithi100 bash[82644]: cluster 2024-07-28T19:40:54.681346+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-07-28T19:40:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:55 smithi100 bash[82644]: audit 2024-07-28T19:40:54.682902+0000 mon.a (mon.0) 741 : audit [INF] from='client.? 172.21.15.100:0/801420757' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:55 smithi100 bash[82644]: audit 2024-07-28T19:40:54.868768+0000 mon.a (mon.0) 742 : audit [INF] from='client.? 172.21.15.100:0/801420757' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:55.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:55 smithi100 bash[82644]: cluster 2024-07-28T19:40:54.869338+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-07-28T19:40:56.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:55 smithi138 bash[67894]: cluster 2024-07-28T19:40:54.681346+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-07-28T19:40:56.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:55 smithi138 bash[67894]: audit 2024-07-28T19:40:54.682902+0000 mon.a (mon.0) 741 : audit [INF] from='client.? 172.21.15.100:0/801420757' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:40:56.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:55 smithi138 bash[67894]: audit 2024-07-28T19:40:54.868768+0000 mon.a (mon.0) 742 : audit [INF] from='client.? 172.21.15.100:0/801420757' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:40:56.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:55 smithi138 bash[67894]: cluster 2024-07-28T19:40:54.869338+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-07-28T19:40:57.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:56 smithi100 bash[80199]: cluster 2024-07-28T19:40:55.665470+0000 mgr.y (mgr.25358) 232 : cluster [DBG] pgmap v438: 201 pgs: 29 unknown, 172 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.2 KiB/s wr, 44 op/s 2024-07-28T19:40:57.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:56 smithi100 bash[80199]: cluster 2024-07-28T19:40:55.874335+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-07-28T19:40:57.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:56 smithi100 bash[82644]: cluster 2024-07-28T19:40:55.665470+0000 mgr.y (mgr.25358) 232 : cluster [DBG] pgmap v438: 201 pgs: 29 unknown, 172 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.2 KiB/s wr, 44 op/s 2024-07-28T19:40:57.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:56 smithi100 bash[82644]: cluster 2024-07-28T19:40:55.874335+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-07-28T19:40:57.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:56 smithi138 bash[67894]: cluster 2024-07-28T19:40:55.665470+0000 mgr.y (mgr.25358) 232 : cluster [DBG] pgmap v438: 201 pgs: 29 unknown, 172 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.2 KiB/s wr, 44 op/s 2024-07-28T19:40:57.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:56 smithi138 bash[67894]: cluster 2024-07-28T19:40:55.874335+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-07-28T19:40:57.952 INFO:tasks.workunit.client.0.smithi100.stdout:image: image22 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image22 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:found image22 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:erased image22 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:image: image22 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:image: image23 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image22 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:found image22 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:erased image22 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image23 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:found image23 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:erased image23 2024-07-28T19:40:57.953 INFO:tasks.workunit.client.0.smithi100.stdout:100.00% done 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:image: image22 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:image: image23 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:image: image24 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image22 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:found image22 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:erased image22 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image23 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:found image23 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:erased image23 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image24 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:found image24 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:erased image24 2024-07-28T19:40:57.954 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCopy (4074 ms) 2024-07-28T19:40:57.955 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-07-28T19:40:58.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:57 smithi100 bash[80199]: cluster 2024-07-28T19:40:56.884452+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-07-28T19:40:58.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:57 smithi100 bash[82644]: cluster 2024-07-28T19:40:56.884452+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-07-28T19:40:58.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:57 smithi138 bash[67894]: cluster 2024-07-28T19:40:56.884452+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-07-28T19:40:59.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:58 smithi100 bash[80199]: cluster 2024-07-28T19:40:57.666253+0000 mgr.y (mgr.25358) 233 : cluster [DBG] pgmap v441: 201 pgs: 29 unknown, 172 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 KiB/s wr, 3 op/s 2024-07-28T19:40:59.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:58 smithi100 bash[82644]: cluster 2024-07-28T19:40:57.666253+0000 mgr.y (mgr.25358) 233 : cluster [DBG] pgmap v441: 201 pgs: 29 unknown, 172 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 KiB/s wr, 3 op/s 2024-07-28T19:40:59.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:58 smithi138 bash[67894]: cluster 2024-07-28T19:40:57.666253+0000 mgr.y (mgr.25358) 233 : cluster [DBG] pgmap v441: 201 pgs: 29 unknown, 172 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 KiB/s wr, 3 op/s 2024-07-28T19:40:59.866 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:59 smithi100 bash[80199]: cluster 2024-07-28T19:40:58.903367+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-07-28T19:41:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:59 smithi100 bash[80199]: audit 2024-07-28T19:40:58.904721+0000 mon.a (mon.0) 747 : audit [INF] from='client.? 172.21.15.100:0/2895840833' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:59 smithi100 bash[80199]: audit 2024-07-28T19:40:59.864489+0000 mon.a (mon.0) 748 : audit [INF] from='client.? 172.21.15.100:0/2895840833' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:00.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:40:59 smithi100 bash[80199]: cluster 2024-07-28T19:40:59.865071+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-07-28T19:41:00.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:59 smithi100 bash[82644]: cluster 2024-07-28T19:40:58.903367+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-07-28T19:41:00.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:59 smithi100 bash[82644]: audit 2024-07-28T19:40:58.904721+0000 mon.a (mon.0) 747 : audit [INF] from='client.? 172.21.15.100:0/2895840833' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:00.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:59 smithi100 bash[82644]: audit 2024-07-28T19:40:59.864489+0000 mon.a (mon.0) 748 : audit [INF] from='client.? 172.21.15.100:0/2895840833' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:00.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:40:59 smithi100 bash[82644]: cluster 2024-07-28T19:40:59.865071+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-07-28T19:41:00.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:59 smithi138 bash[67894]: cluster 2024-07-28T19:40:58.903367+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-07-28T19:41:00.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:59 smithi138 bash[67894]: audit 2024-07-28T19:40:58.904721+0000 mon.a (mon.0) 747 : audit [INF] from='client.? 172.21.15.100:0/2895840833' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:00.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:59 smithi138 bash[67894]: audit 2024-07-28T19:40:59.864489+0000 mon.a (mon.0) 748 : audit [INF] from='client.? 172.21.15.100:0/2895840833' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:00.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:40:59 smithi138 bash[67894]: cluster 2024-07-28T19:40:59.865071+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-07-28T19:41:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:59.667433+0000 mgr.y (mgr.25358) 234 : cluster [DBG] pgmap v443: 233 pgs: 8 creating+peering, 24 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 307 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-07-28T19:41:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:00 smithi100 bash[80199]: cluster 2024-07-28T19:40:59.898569+0000 mon.a (mon.0) 750 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:41:01.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:00 smithi100 bash[80199]: cluster 2024-07-28T19:41:00.868968+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-07-28T19:41:01.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:59.667433+0000 mgr.y (mgr.25358) 234 : cluster [DBG] pgmap v443: 233 pgs: 8 creating+peering, 24 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 307 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-07-28T19:41:01.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:00 smithi100 bash[82644]: cluster 2024-07-28T19:40:59.898569+0000 mon.a (mon.0) 750 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:41:01.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:00 smithi100 bash[82644]: cluster 2024-07-28T19:41:00.868968+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-07-28T19:41:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:59.667433+0000 mgr.y (mgr.25358) 234 : cluster [DBG] pgmap v443: 233 pgs: 8 creating+peering, 24 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 307 KiB/s rd, 21 KiB/s wr, 365 op/s 2024-07-28T19:41:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:00 smithi138 bash[67894]: cluster 2024-07-28T19:40:59.898569+0000 mon.a (mon.0) 750 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:41:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:00 smithi138 bash[67894]: cluster 2024-07-28T19:41:00.868968+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-07-28T19:41:02.637 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 1 2024-07-28T19:41:02.637 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:41:02.637 INFO:tasks.workunit.client.0.smithi100.stdout:image: image25 2024-07-28T19:41:02.637 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image25 2024-07-28T19:41:02.689 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 2 2024-07-28T19:41:02.690 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:41:02.690 INFO:tasks.workunit.client.0.smithi100.stdout:image: image25 2024-07-28T19:41:02.690 INFO:tasks.workunit.client.0.smithi100.stdout:image: image26 2024-07-28T19:41:02.690 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image25 2024-07-28T19:41:02.690 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image26 2024-07-28T19:41:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:41:02 smithi138 bash[60615]: ts=2024-07-28T19:41:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:41:02.824 INFO:tasks.workunit.client.0.smithi100.stdout:100.00% done 2024-07-28T19:41:02.824 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 3 2024-07-28T19:41:02.824 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:41:02.824 INFO:tasks.workunit.client.0.smithi100.stdout:image: image25 2024-07-28T19:41:02.824 INFO:tasks.workunit.client.0.smithi100.stdout:image: image26 2024-07-28T19:41:02.825 INFO:tasks.workunit.client.0.smithi100.stdout:image: image27 2024-07-28T19:41:02.825 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image25 2024-07-28T19:41:02.825 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image26 2024-07-28T19:41:02.825 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image27 2024-07-28T19:41:02.916 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCopyPP (4964 ms) 2024-07-28T19:41:02.916 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-07-28T19:41:03.246 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:41:02] "GET /metrics HTTP/1.1" 200 41274 "" "Prometheus/2.43.0" 2024-07-28T19:41:03.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:02 smithi100 bash[80199]: cluster 2024-07-28T19:41:01.668427+0000 mgr.y (mgr.25358) 235 : cluster [DBG] pgmap v446: 233 pgs: 8 creating+peering, 23 unknown, 202 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 306 KiB/s rd, 20 KiB/s wr, 364 op/s 2024-07-28T19:41:03.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:02 smithi100 bash[80199]: cluster 2024-07-28T19:41:01.869796+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-07-28T19:41:03.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:02 smithi100 bash[80199]: cluster 2024-07-28T19:41:01.905004+0000 mon.a (mon.0) 753 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:41:03.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:02 smithi100 bash[82644]: cluster 2024-07-28T19:41:01.668427+0000 mgr.y (mgr.25358) 235 : cluster [DBG] pgmap v446: 233 pgs: 8 creating+peering, 23 unknown, 202 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 306 KiB/s rd, 20 KiB/s wr, 364 op/s 2024-07-28T19:41:03.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:02 smithi100 bash[82644]: cluster 2024-07-28T19:41:01.869796+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-07-28T19:41:03.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:02 smithi100 bash[82644]: cluster 2024-07-28T19:41:01.905004+0000 mon.a (mon.0) 753 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:41:03.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:02 smithi138 bash[67894]: cluster 2024-07-28T19:41:01.668427+0000 mgr.y (mgr.25358) 235 : cluster [DBG] pgmap v446: 233 pgs: 8 creating+peering, 23 unknown, 202 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 306 KiB/s rd, 20 KiB/s wr, 364 op/s 2024-07-28T19:41:03.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:02 smithi138 bash[67894]: cluster 2024-07-28T19:41:01.869796+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-07-28T19:41:03.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:02 smithi138 bash[67894]: cluster 2024-07-28T19:41:01.905004+0000 mon.a (mon.0) 753 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:41:04.884 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[80199]: cluster 2024-07-28T19:41:03.670317+0000 mgr.y (mgr.25358) 236 : cluster [DBG] pgmap v448: 233 pgs: 8 creating+peering, 225 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 295 KiB/s rd, 18 KiB/s wr, 343 op/s 2024-07-28T19:41:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[80199]: cluster 2024-07-28T19:41:03.890291+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-07-28T19:41:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[80199]: audit 2024-07-28T19:41:03.897834+0000 mon.a (mon.0) 755 : audit [INF] from='client.? 172.21.15.100:0/2406874592' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[80199]: audit 2024-07-28T19:41:04.871726+0000 mon.a (mon.0) 756 : audit [INF] from='client.? 172.21.15.100:0/2406874592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:05.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[80199]: cluster 2024-07-28T19:41:04.872367+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-07-28T19:41:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[82644]: cluster 2024-07-28T19:41:03.670317+0000 mgr.y (mgr.25358) 236 : cluster [DBG] pgmap v448: 233 pgs: 8 creating+peering, 225 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 295 KiB/s rd, 18 KiB/s wr, 343 op/s 2024-07-28T19:41:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[82644]: cluster 2024-07-28T19:41:03.890291+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-07-28T19:41:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[82644]: audit 2024-07-28T19:41:03.897834+0000 mon.a (mon.0) 755 : audit [INF] from='client.? 172.21.15.100:0/2406874592' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[82644]: audit 2024-07-28T19:41:04.871726+0000 mon.a (mon.0) 756 : audit [INF] from='client.? 172.21.15.100:0/2406874592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:05.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:04 smithi100 bash[82644]: cluster 2024-07-28T19:41:04.872367+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-07-28T19:41:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:04 smithi138 bash[67894]: cluster 2024-07-28T19:41:03.670317+0000 mgr.y (mgr.25358) 236 : cluster [DBG] pgmap v448: 233 pgs: 8 creating+peering, 225 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 295 KiB/s rd, 18 KiB/s wr, 343 op/s 2024-07-28T19:41:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:04 smithi138 bash[67894]: cluster 2024-07-28T19:41:03.890291+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-07-28T19:41:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:04 smithi138 bash[67894]: audit 2024-07-28T19:41:03.897834+0000 mon.a (mon.0) 755 : audit [INF] from='client.? 172.21.15.100:0/2406874592' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:04 smithi138 bash[67894]: audit 2024-07-28T19:41:04.871726+0000 mon.a (mon.0) 756 : audit [INF] from='client.? 172.21.15.100:0/2406874592' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:04 smithi138 bash[67894]: cluster 2024-07-28T19:41:04.872367+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-07-28T19:41:07.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:06 smithi100 bash[80199]: cluster 2024-07-28T19:41:05.671392+0000 mgr.y (mgr.25358) 237 : cluster [DBG] pgmap v451: 265 pgs: 5 creating+peering, 20 unknown, 240 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 284 KiB/s rd, 18 KiB/s wr, 331 op/s 2024-07-28T19:41:07.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:06 smithi100 bash[80199]: cluster 2024-07-28T19:41:05.910928+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-07-28T19:41:07.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:06 smithi100 bash[82644]: cluster 2024-07-28T19:41:05.671392+0000 mgr.y (mgr.25358) 237 : cluster [DBG] pgmap v451: 265 pgs: 5 creating+peering, 20 unknown, 240 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 284 KiB/s rd, 18 KiB/s wr, 331 op/s 2024-07-28T19:41:07.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:06 smithi100 bash[82644]: cluster 2024-07-28T19:41:05.910928+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-07-28T19:41:07.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:06 smithi138 bash[67894]: cluster 2024-07-28T19:41:05.671392+0000 mgr.y (mgr.25358) 237 : cluster [DBG] pgmap v451: 265 pgs: 5 creating+peering, 20 unknown, 240 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 284 KiB/s rd, 18 KiB/s wr, 331 op/s 2024-07-28T19:41:07.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:06 smithi138 bash[67894]: cluster 2024-07-28T19:41:05.910928+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-07-28T19:41:08.238 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:07 smithi100 bash[82644]: cluster 2024-07-28T19:41:06.916213+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-07-28T19:41:08.239 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:07 smithi100 bash[82644]: audit 2024-07-28T19:41:07.632905+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:08.239 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:07 smithi100 bash[80199]: cluster 2024-07-28T19:41:06.916213+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-07-28T19:41:08.239 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:07 smithi100 bash[80199]: audit 2024-07-28T19:41:07.632905+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:07 smithi138 bash[67894]: cluster 2024-07-28T19:41:06.916213+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-07-28T19:41:08.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:07 smithi138 bash[67894]: audit 2024-07-28T19:41:07.632905+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:09.162 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch upgrade resume' 2024-07-28T19:41:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:08 smithi100 bash[80199]: cluster 2024-07-28T19:41:07.687177+0000 mgr.y (mgr.25358) 238 : cluster [DBG] pgmap v454: 265 pgs: 5 creating+peering, 20 unknown, 240 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 1.5 KiB/s wr, 24 op/s 2024-07-28T19:41:09.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:08 smithi100 bash[80199]: cluster 2024-07-28T19:41:07.928715+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-07-28T19:41:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:08 smithi100 bash[82644]: cluster 2024-07-28T19:41:07.687177+0000 mgr.y (mgr.25358) 238 : cluster [DBG] pgmap v454: 265 pgs: 5 creating+peering, 20 unknown, 240 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 1.5 KiB/s wr, 24 op/s 2024-07-28T19:41:09.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:08 smithi100 bash[82644]: cluster 2024-07-28T19:41:07.928715+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-07-28T19:41:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:08 smithi138 bash[67894]: cluster 2024-07-28T19:41:07.687177+0000 mgr.y (mgr.25358) 238 : cluster [DBG] pgmap v454: 265 pgs: 5 creating+peering, 20 unknown, 240 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 1.5 KiB/s wr, 24 op/s 2024-07-28T19:41:09.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:08 smithi138 bash[67894]: cluster 2024-07-28T19:41:07.928715+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-07-28T19:41:10.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:09 smithi100 bash[80199]: cluster 2024-07-28T19:41:08.931351+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-07-28T19:41:10.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:09 smithi100 bash[80199]: cluster 2024-07-28T19:41:09.870968+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-07-28T19:41:10.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:09 smithi100 bash[82644]: cluster 2024-07-28T19:41:08.931351+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-07-28T19:41:10.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:09 smithi100 bash[82644]: cluster 2024-07-28T19:41:09.870968+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-07-28T19:41:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:09 smithi138 bash[67894]: cluster 2024-07-28T19:41:08.931351+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-07-28T19:41:10.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:09 smithi138 bash[67894]: cluster 2024-07-28T19:41:09.870968+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-07-28T19:41:11.149 INFO:tasks.workunit.client.0.smithi100.stdout:image: image28 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image28 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:found image28 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:erased image28 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:image: image28 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:image: image29 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image28 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:found image28 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:erased image28 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image29 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:found image29 2024-07-28T19:41:11.150 INFO:tasks.workunit.client.0.smithi100.stdout:erased image29 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:100.00% done 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:image: image28 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:image: image29 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:image: image30 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image28 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:found image28 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:erased image28 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image29 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:found image29 2024-07-28T19:41:11.151 INFO:tasks.workunit.client.0.smithi100.stdout:erased image29 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image30 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:found image30 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:erased image30 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:image: image28 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:image: image29 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:image: image30 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:image: image31 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image28 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:found image28 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:erased image28 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image29 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:found image29 2024-07-28T19:41:11.152 INFO:tasks.workunit.client.0.smithi100.stdout:erased image29 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image30 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:found image30 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:erased image30 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image31 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:found image31 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:erased image31 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:image: image28 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:image: image29 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:image: image30 2024-07-28T19:41:11.153 INFO:tasks.workunit.client.0.smithi100.stdout:image: image31 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:image: image32 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image28 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:found image28 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:erased image28 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image29 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:found image29 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:erased image29 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image30 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:found image30 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:erased image30 2024-07-28T19:41:11.154 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image31 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:found image31 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:erased image31 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image32 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:found image32 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:erased image32 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:100.00% done 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:image: image28 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:image: image29 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:image: image30 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:image: image31 2024-07-28T19:41:11.155 INFO:tasks.workunit.client.0.smithi100.stdout:image: image32 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:image: image33 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image28 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:found image28 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:erased image28 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image29 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:found image29 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:erased image29 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image30 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:found image30 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:erased image30 2024-07-28T19:41:11.156 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image31 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:found image31 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:erased image31 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image32 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:found image32 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:erased image32 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image33 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:found image33 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:erased image33 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestDeepCopy (8233 ms) 2024-07-28T19:41:11.157 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-07-28T19:41:11.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:10 smithi100 bash[80199]: cluster 2024-07-28T19:41:09.688346+0000 mgr.y (mgr.25358) 239 : cluster [DBG] pgmap v457: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-07-28T19:41:11.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:10 smithi100 bash[82644]: cluster 2024-07-28T19:41:09.688346+0000 mgr.y (mgr.25358) 239 : cluster [DBG] pgmap v457: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-07-28T19:41:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:10 smithi138 bash[67894]: cluster 2024-07-28T19:41:09.688346+0000 mgr.y (mgr.25358) 239 : cluster [DBG] pgmap v457: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-07-28T19:41:12.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:11 smithi100 bash[80199]: cluster 2024-07-28T19:41:10.964595+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-07-28T19:41:12.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:11 smithi100 bash[82644]: cluster 2024-07-28T19:41:10.964595+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-07-28T19:41:12.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:11 smithi138 bash[67894]: cluster 2024-07-28T19:41:10.964595+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-07-28T19:41:12.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:41:12 smithi138 bash[60615]: ts=2024-07-28T19:41:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:41:12.989 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:13.246 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:41:12] "GET /metrics HTTP/1.1" 200 48521 "" "Prometheus/2.43.0" 2024-07-28T19:41:13.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:12 smithi100 bash[80199]: cluster 2024-07-28T19:41:11.688819+0000 mgr.y (mgr.25358) 240 : cluster [DBG] pgmap v460: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 431 KiB/s rd, 26 KiB/s wr, 512 op/s 2024-07-28T19:41:13.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:12 smithi100 bash[80199]: cluster 2024-07-28T19:41:11.971852+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-07-28T19:41:13.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:12 smithi100 bash[80199]: audit 2024-07-28T19:41:11.973737+0000 mon.a (mon.0) 766 : audit [INF] from='client.? 172.21.15.100:0/3971821229' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:13.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:12 smithi100 bash[82644]: cluster 2024-07-28T19:41:11.688819+0000 mgr.y (mgr.25358) 240 : cluster [DBG] pgmap v460: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 431 KiB/s rd, 26 KiB/s wr, 512 op/s 2024-07-28T19:41:13.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:12 smithi100 bash[82644]: cluster 2024-07-28T19:41:11.971852+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-07-28T19:41:13.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:12 smithi100 bash[82644]: audit 2024-07-28T19:41:11.973737+0000 mon.a (mon.0) 766 : audit [INF] from='client.? 172.21.15.100:0/3971821229' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:12 smithi138 bash[67894]: cluster 2024-07-28T19:41:11.688819+0000 mgr.y (mgr.25358) 240 : cluster [DBG] pgmap v460: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 431 KiB/s rd, 26 KiB/s wr, 512 op/s 2024-07-28T19:41:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:12 smithi138 bash[67894]: cluster 2024-07-28T19:41:11.971852+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-07-28T19:41:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:12 smithi138 bash[67894]: audit 2024-07-28T19:41:11.973737+0000 mon.a (mon.0) 766 : audit [INF] from='client.? 172.21.15.100:0/3971821229' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:13.937 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:41:14.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:13 smithi100 bash[80199]: audit 2024-07-28T19:41:12.976553+0000 mon.a (mon.0) 767 : audit [INF] from='client.? 172.21.15.100:0/3971821229' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:14.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:13 smithi100 bash[80199]: cluster 2024-07-28T19:41:12.977199+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-07-28T19:41:14.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:13 smithi100 bash[82644]: audit 2024-07-28T19:41:12.976553+0000 mon.a (mon.0) 767 : audit [INF] from='client.? 172.21.15.100:0/3971821229' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:14.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:13 smithi100 bash[82644]: cluster 2024-07-28T19:41:12.977199+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-07-28T19:41:14.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:13 smithi138 bash[67894]: audit 2024-07-28T19:41:12.976553+0000 mon.a (mon.0) 767 : audit [INF] from='client.? 172.21.15.100:0/3971821229' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:14.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:13 smithi138 bash[67894]: cluster 2024-07-28T19:41:12.977199+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-07-28T19:41:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:14 smithi100 bash[82644]: cluster 2024-07-28T19:41:13.690893+0000 mgr.y (mgr.25358) 241 : cluster [DBG] pgmap v463: 297 pgs: 8 unknown, 289 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 447 KiB/s rd, 27 KiB/s wr, 525 op/s 2024-07-28T19:41:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:14 smithi100 bash[82644]: cluster 2024-07-28T19:41:13.996982+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-07-28T19:41:15.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:14 smithi100 bash[82644]: cluster 2024-07-28T19:41:14.872447+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-07-28T19:41:15.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:14 smithi100 bash[80199]: cluster 2024-07-28T19:41:13.690893+0000 mgr.y (mgr.25358) 241 : cluster [DBG] pgmap v463: 297 pgs: 8 unknown, 289 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 447 KiB/s rd, 27 KiB/s wr, 525 op/s 2024-07-28T19:41:15.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:14 smithi100 bash[80199]: cluster 2024-07-28T19:41:13.996982+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-07-28T19:41:15.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:14 smithi100 bash[80199]: cluster 2024-07-28T19:41:14.872447+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-07-28T19:41:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:14 smithi138 bash[67894]: cluster 2024-07-28T19:41:13.690893+0000 mgr.y (mgr.25358) 241 : cluster [DBG] pgmap v463: 297 pgs: 8 unknown, 289 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 447 KiB/s rd, 27 KiB/s wr, 525 op/s 2024-07-28T19:41:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:14 smithi138 bash[67894]: cluster 2024-07-28T19:41:13.996982+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-07-28T19:41:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:14 smithi138 bash[67894]: cluster 2024-07-28T19:41:14.872447+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-07-28T19:41:15.506 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 1 2024-07-28T19:41:15.507 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:41:15.507 INFO:tasks.workunit.client.0.smithi100.stdout:image: image34 2024-07-28T19:41:15.507 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image34 2024-07-28T19:41:15.559 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 2 2024-07-28T19:41:15.560 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:41:15.560 INFO:tasks.workunit.client.0.smithi100.stdout:image: image34 2024-07-28T19:41:15.560 INFO:tasks.workunit.client.0.smithi100.stdout:image: image35 2024-07-28T19:41:15.560 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image34 2024-07-28T19:41:15.560 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image35 2024-07-28T19:41:15.703 INFO:tasks.workunit.client.0.smithi100.stdout:100.00% done 2024-07-28T19:41:15.703 INFO:tasks.workunit.client.0.smithi100.stdout:num images is: 3 2024-07-28T19:41:15.703 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:41:15.704 INFO:tasks.workunit.client.0.smithi100.stdout:image: image34 2024-07-28T19:41:15.704 INFO:tasks.workunit.client.0.smithi100.stdout:image: image35 2024-07-28T19:41:15.704 INFO:tasks.workunit.client.0.smithi100.stdout:image: image36 2024-07-28T19:41:15.704 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image34 2024-07-28T19:41:15.704 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image35 2024-07-28T19:41:15.704 INFO:tasks.workunit.client.0.smithi100.stdout:expected = image36 2024-07-28T19:41:15.798 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4648 ms) 2024-07-28T19:41:15.798 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-07-28T19:41:15.798 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:17.293 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:17 smithi100 bash[80199]: cluster 2024-07-28T19:41:15.692122+0000 mgr.y (mgr.25358) 242 : cluster [DBG] pgmap v466: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.0 KiB/s wr, 138 op/s 2024-07-28T19:41:17.293 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:17 smithi100 bash[80199]: cluster 2024-07-28T19:41:16.006983+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-07-28T19:41:17.293 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:17 smithi100 bash[82644]: cluster 2024-07-28T19:41:15.692122+0000 mgr.y (mgr.25358) 242 : cluster [DBG] pgmap v466: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.0 KiB/s wr, 138 op/s 2024-07-28T19:41:17.293 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:17 smithi100 bash[82644]: cluster 2024-07-28T19:41:16.006983+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-07-28T19:41:17.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:17 smithi138 bash[67894]: cluster 2024-07-28T19:41:15.692122+0000 mgr.y (mgr.25358) 242 : cluster [DBG] pgmap v466: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.0 KiB/s wr, 138 op/s 2024-07-28T19:41:17.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:17 smithi138 bash[67894]: cluster 2024-07-28T19:41:16.006983+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-07-28T19:41:17.607 INFO:teuthology.orchestra.run.smithi100.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:41:18.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:18 smithi138 bash[67894]: cluster 2024-07-28T19:41:17.022263+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-07-28T19:41:18.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:18 smithi138 bash[67894]: audit 2024-07-28T19:41:17.610607+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:18.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:18 smithi138 bash[67894]: audit 2024-07-28T19:41:17.611869+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:18.387 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:18 smithi100 bash[80199]: cluster 2024-07-28T19:41:17.022263+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-07-28T19:41:18.387 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:18 smithi100 bash[80199]: audit 2024-07-28T19:41:17.610607+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:18.387 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:18 smithi100 bash[80199]: audit 2024-07-28T19:41:17.611869+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:18.388 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:18 smithi100 bash[82644]: cluster 2024-07-28T19:41:17.022263+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-07-28T19:41:18.388 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:18 smithi100 bash[82644]: audit 2024-07-28T19:41:17.610607+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:18.388 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:18 smithi100 bash[82644]: audit 2024-07-28T19:41:17.611869+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:18.389 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'echo wait for half of osds to upgrade' 2024-07-28T19:41:19.082 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 1 2024-07-28T19:41:19.082 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:found snap1 with size 2097152 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 2 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:found snap1 with size 2097152 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:found snap2 with size 4194304 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 1 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:41:19.083 INFO:tasks.workunit.client.0.smithi100.stdout:found snap2 with size 4194304 2024-07-28T19:41:19.084 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 0 2024-07-28T19:41:19.084 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:41:19.084 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (3285 ms) 2024-07-28T19:41:19.084 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-07-28T19:41:19.084 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:19 smithi138 bash[67894]: audit 2024-07-28T19:41:17.605096+0000 mgr.y (mgr.25358) 243 : audit [DBG] from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:41:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:19 smithi138 bash[67894]: cephadm 2024-07-28T19:41:17.605664+0000 mgr.y (mgr.25358) 244 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:41:19.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:19 smithi138 bash[67894]: cluster 2024-07-28T19:41:17.692620+0000 mgr.y (mgr.25358) 245 : cluster [DBG] pgmap v469: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 511 B/s wr, 35 op/s 2024-07-28T19:41:19.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:19 smithi138 bash[67894]: cluster 2024-07-28T19:41:18.027194+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-07-28T19:41:19.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:19 smithi100 bash[80199]: audit 2024-07-28T19:41:17.605096+0000 mgr.y (mgr.25358) 243 : audit [DBG] from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:41:19.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:19 smithi100 bash[80199]: cephadm 2024-07-28T19:41:17.605664+0000 mgr.y (mgr.25358) 244 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:41:19.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:19 smithi100 bash[80199]: cluster 2024-07-28T19:41:17.692620+0000 mgr.y (mgr.25358) 245 : cluster [DBG] pgmap v469: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 511 B/s wr, 35 op/s 2024-07-28T19:41:19.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:19 smithi100 bash[80199]: cluster 2024-07-28T19:41:18.027194+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-07-28T19:41:19.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:19 smithi100 bash[82644]: audit 2024-07-28T19:41:17.605096+0000 mgr.y (mgr.25358) 243 : audit [DBG] from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:41:19.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:19 smithi100 bash[82644]: cephadm 2024-07-28T19:41:17.605664+0000 mgr.y (mgr.25358) 244 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:41:19.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:19 smithi100 bash[82644]: cluster 2024-07-28T19:41:17.692620+0000 mgr.y (mgr.25358) 245 : cluster [DBG] pgmap v469: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 511 B/s wr, 35 op/s 2024-07-28T19:41:19.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:19 smithi100 bash[82644]: cluster 2024-07-28T19:41:18.027194+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-07-28T19:41:20.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:20 smithi138 bash[67894]: cluster 2024-07-28T19:41:19.039425+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-07-28T19:41:20.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:20 smithi100 bash[80199]: cluster 2024-07-28T19:41:19.039425+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-07-28T19:41:20.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:20 smithi100 bash[82644]: cluster 2024-07-28T19:41:19.039425+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-07-28T19:41:21.118 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (2036 ms) 2024-07-28T19:41:21.119 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-07-28T19:41:21.119 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:21.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:21 smithi100 bash[80199]: cluster 2024-07-28T19:41:19.693751+0000 mgr.y (mgr.25358) 246 : cluster [DBG] pgmap v472: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 383 KiB/s rd, 27 KiB/s wr, 460 op/s 2024-07-28T19:41:21.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:21 smithi100 bash[80199]: cluster 2024-07-28T19:41:20.061019+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-07-28T19:41:21.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:21 smithi100 bash[82644]: cluster 2024-07-28T19:41:19.693751+0000 mgr.y (mgr.25358) 246 : cluster [DBG] pgmap v472: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 383 KiB/s rd, 27 KiB/s wr, 460 op/s 2024-07-28T19:41:21.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:21 smithi100 bash[82644]: cluster 2024-07-28T19:41:20.061019+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-07-28T19:41:21.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:21 smithi138 bash[67894]: cluster 2024-07-28T19:41:19.693751+0000 mgr.y (mgr.25358) 246 : cluster [DBG] pgmap v472: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 383 KiB/s rd, 27 KiB/s wr, 460 op/s 2024-07-28T19:41:21.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:21 smithi138 bash[67894]: cluster 2024-07-28T19:41:20.061019+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-07-28T19:41:22.114 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 1 2024-07-28T19:41:22.114 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:41:22.114 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:41:22.115 INFO:tasks.workunit.client.0.smithi100.stdout:found snap1 with size 2097152 2024-07-28T19:41:22.463 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:41:22.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:22 smithi100 bash[80199]: cluster 2024-07-28T19:41:21.064892+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-07-28T19:41:22.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:22 smithi100 bash[82644]: cluster 2024-07-28T19:41:21.064892+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-07-28T19:41:22.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:22 smithi138 bash[67894]: cluster 2024-07-28T19:41:21.064892+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-07-28T19:41:22.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:41:22 smithi138 bash[60615]: ts=2024-07-28T19:41:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:41:23.129 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 2 2024-07-28T19:41:23.129 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:41:23.129 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:41:23.129 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:41:23.129 INFO:tasks.workunit.client.0.smithi100.stdout:found snap1 with size 2097152 2024-07-28T19:41:23.129 INFO:tasks.workunit.client.0.smithi100.stdout:found snap2 with size 4194304 2024-07-28T19:41:23.246 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:41:22] "GET /metrics HTTP/1.1" 200 50982 "" "Prometheus/2.43.0" 2024-07-28T19:41:23.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:23 smithi100 bash[80199]: cluster 2024-07-28T19:41:21.694626+0000 mgr.y (mgr.25358) 247 : cluster [DBG] pgmap v475: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 27 KiB/s wr, 461 op/s 2024-07-28T19:41:23.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:23 smithi100 bash[80199]: cluster 2024-07-28T19:41:22.087654+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-07-28T19:41:23.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:23 smithi100 bash[80199]: audit 2024-07-28T19:41:22.632932+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:23.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:23 smithi100 bash[80199]: audit 2024-07-28T19:41:22.772410+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:23.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:23 smithi100 bash[82644]: cluster 2024-07-28T19:41:21.694626+0000 mgr.y (mgr.25358) 247 : cluster [DBG] pgmap v475: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 27 KiB/s wr, 461 op/s 2024-07-28T19:41:23.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:23 smithi100 bash[82644]: cluster 2024-07-28T19:41:22.087654+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-07-28T19:41:23.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:23 smithi100 bash[82644]: audit 2024-07-28T19:41:22.632932+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:23.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:23 smithi100 bash[82644]: audit 2024-07-28T19:41:22.772410+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:23 smithi138 bash[67894]: cluster 2024-07-28T19:41:21.694626+0000 mgr.y (mgr.25358) 247 : cluster [DBG] pgmap v475: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 27 KiB/s wr, 461 op/s 2024-07-28T19:41:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:23 smithi138 bash[67894]: cluster 2024-07-28T19:41:22.087654+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-07-28T19:41:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:23 smithi138 bash[67894]: audit 2024-07-28T19:41:22.632932+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:23 smithi138 bash[67894]: audit 2024-07-28T19:41:22.772410+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.128 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 1 2024-07-28T19:41:24.128 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:41:24.128 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:41:24.128 INFO:tasks.workunit.client.0.smithi100.stdout:found snap2 with size 4194304 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: cluster 2024-07-28T19:41:23.092326+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.145323+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.152321+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.559285+0000 mon.a (mon.0) 785 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.560342+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.569851+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.654029+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.659380+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.661853+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.663635+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.175 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[82644]: audit 2024-07-28T19:41:23.665947+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: cluster 2024-07-28T19:41:23.092326+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.145323+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.152321+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.559285+0000 mon.a (mon.0) 785 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.560342+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.569851+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.654029+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.659380+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.661853+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.663635+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:24 smithi100 bash[80199]: audit 2024-07-28T19:41:23.665947+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-28T19:41:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: cluster 2024-07-28T19:41:23.092326+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.145323+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.152321+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.559285+0000 mon.a (mon.0) 785 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.560342+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.569851+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.654029+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.659380+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.661853+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.663635+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:24.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:24 smithi138 bash[67894]: audit 2024-07-28T19:41:23.665947+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-28T19:41:24.891 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 0 2024-07-28T19:41:24.892 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:41:24.909 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (3790 ms) 2024-07-28T19:41:24.909 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-07-28T19:41:24.909 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:25.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[80199]: audit 2024-07-28T19:41:23.666341+0000 mgr.y (mgr.25358) 248 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-28T19:41:25.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[80199]: cephadm 2024-07-28T19:41:23.669251+0000 mgr.y (mgr.25358) 249 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-07-28T19:41:25.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[80199]: cluster 2024-07-28T19:41:23.696215+0000 mgr.y (mgr.25358) 250 : cluster [DBG] pgmap v478: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 5.2 KiB/s wr, 69 op/s 2024-07-28T19:41:25.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[80199]: cluster 2024-07-28T19:41:24.103871+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-07-28T19:41:25.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[80199]: cluster 2024-07-28T19:41:24.878185+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-07-28T19:41:25.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[82644]: audit 2024-07-28T19:41:23.666341+0000 mgr.y (mgr.25358) 248 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-28T19:41:25.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[82644]: cephadm 2024-07-28T19:41:23.669251+0000 mgr.y (mgr.25358) 249 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-07-28T19:41:25.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[82644]: cluster 2024-07-28T19:41:23.696215+0000 mgr.y (mgr.25358) 250 : cluster [DBG] pgmap v478: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 5.2 KiB/s wr, 69 op/s 2024-07-28T19:41:25.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[82644]: cluster 2024-07-28T19:41:24.103871+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-07-28T19:41:25.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:25 smithi100 bash[82644]: cluster 2024-07-28T19:41:24.878185+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-07-28T19:41:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:25 smithi138 bash[67894]: audit 2024-07-28T19:41:23.666341+0000 mgr.y (mgr.25358) 248 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-07-28T19:41:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:25 smithi138 bash[67894]: cephadm 2024-07-28T19:41:23.669251+0000 mgr.y (mgr.25358) 249 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-07-28T19:41:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:25 smithi138 bash[67894]: cluster 2024-07-28T19:41:23.696215+0000 mgr.y (mgr.25358) 250 : cluster [DBG] pgmap v478: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 5.2 KiB/s wr, 69 op/s 2024-07-28T19:41:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:25 smithi138 bash[67894]: cluster 2024-07-28T19:41:24.103871+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-07-28T19:41:25.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:25 smithi138 bash[67894]: cluster 2024-07-28T19:41:24.878185+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-07-28T19:41:27.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:26 smithi100 bash[80199]: cluster 2024-07-28T19:41:25.697382+0000 mgr.y (mgr.25358) 251 : cluster [DBG] pgmap v481: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 83 op/s 2024-07-28T19:41:27.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:26 smithi100 bash[80199]: cluster 2024-07-28T19:41:25.874681+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-07-28T19:41:27.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:26 smithi100 bash[82644]: cluster 2024-07-28T19:41:25.697382+0000 mgr.y (mgr.25358) 251 : cluster [DBG] pgmap v481: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 83 op/s 2024-07-28T19:41:27.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:26 smithi100 bash[82644]: cluster 2024-07-28T19:41:25.874681+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-07-28T19:41:27.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:26 smithi138 bash[67894]: cluster 2024-07-28T19:41:25.697382+0000 mgr.y (mgr.25358) 251 : cluster [DBG] pgmap v481: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 83 op/s 2024-07-28T19:41:27.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:26 smithi138 bash[67894]: cluster 2024-07-28T19:41:25.874681+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-07-28T19:41:27.800 INFO:teuthology.orchestra.run.smithi100.stdout:wait for half of osds to upgrade 2024-07-28T19:41:28.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:27 smithi100 bash[80199]: cluster 2024-07-28T19:41:26.891615+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-07-28T19:41:28.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:27 smithi100 bash[80199]: audit 2024-07-28T19:41:27.703545+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:28.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:27 smithi100 bash[80199]: audit 2024-07-28T19:41:27.705416+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:41:28.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:27 smithi100 bash[80199]: audit 2024-07-28T19:41:27.706774+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:28.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:27 smithi100 bash[82644]: cluster 2024-07-28T19:41:26.891615+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-07-28T19:41:28.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:27 smithi100 bash[82644]: audit 2024-07-28T19:41:27.703545+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:28.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:27 smithi100 bash[82644]: audit 2024-07-28T19:41:27.705416+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:41:28.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:27 smithi100 bash[82644]: audit 2024-07-28T19:41:27.706774+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:28.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:27 smithi138 bash[67894]: cluster 2024-07-28T19:41:26.891615+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-07-28T19:41:28.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:27 smithi138 bash[67894]: audit 2024-07-28T19:41:27.703545+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:28.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:27 smithi138 bash[67894]: audit 2024-07-28T19:41:27.705416+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-07-28T19:41:28.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:27 smithi138 bash[67894]: audit 2024-07-28T19:41:27.706774+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:28.972 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-07-28T19:41:29.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:28 smithi100 bash[80199]: cephadm 2024-07-28T19:41:27.696804+0000 mgr.y (mgr.25358) 252 : cephadm [INF] Upgrade: Updating osd.2 2024-07-28T19:41:29.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:28 smithi100 bash[80199]: cluster 2024-07-28T19:41:27.697899+0000 mgr.y (mgr.25358) 253 : cluster [DBG] pgmap v484: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 1.5 KiB/s wr, 15 op/s 2024-07-28T19:41:29.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:28 smithi100 bash[80199]: cephadm 2024-07-28T19:41:27.710928+0000 mgr.y (mgr.25358) 254 : cephadm [INF] Deploying daemon osd.2 on smithi100 2024-07-28T19:41:29.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:28 smithi100 bash[80199]: cluster 2024-07-28T19:41:27.896754+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-07-28T19:41:29.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:28 smithi100 bash[82644]: cephadm 2024-07-28T19:41:27.696804+0000 mgr.y (mgr.25358) 252 : cephadm [INF] Upgrade: Updating osd.2 2024-07-28T19:41:29.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:28 smithi100 bash[82644]: cluster 2024-07-28T19:41:27.697899+0000 mgr.y (mgr.25358) 253 : cluster [DBG] pgmap v484: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 1.5 KiB/s wr, 15 op/s 2024-07-28T19:41:29.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:28 smithi100 bash[82644]: cephadm 2024-07-28T19:41:27.710928+0000 mgr.y (mgr.25358) 254 : cephadm [INF] Deploying daemon osd.2 on smithi100 2024-07-28T19:41:29.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:28 smithi100 bash[82644]: cluster 2024-07-28T19:41:27.896754+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-07-28T19:41:29.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:28 smithi138 bash[67894]: cephadm 2024-07-28T19:41:27.696804+0000 mgr.y (mgr.25358) 252 : cephadm [INF] Upgrade: Updating osd.2 2024-07-28T19:41:29.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:28 smithi138 bash[67894]: cluster 2024-07-28T19:41:27.697899+0000 mgr.y (mgr.25358) 253 : cluster [DBG] pgmap v484: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 1.5 KiB/s wr, 15 op/s 2024-07-28T19:41:29.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:28 smithi138 bash[67894]: cephadm 2024-07-28T19:41:27.710928+0000 mgr.y (mgr.25358) 254 : cephadm [INF] Deploying daemon osd.2 on smithi100 2024-07-28T19:41:29.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:28 smithi138 bash[67894]: cluster 2024-07-28T19:41:27.896754+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-07-28T19:41:30.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:29 smithi100 bash[82644]: cluster 2024-07-28T19:41:28.910236+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-07-28T19:41:30.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:29 smithi100 bash[80199]: cluster 2024-07-28T19:41:28.910236+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-07-28T19:41:30.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:29 smithi138 bash[67894]: cluster 2024-07-28T19:41:28.910236+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-07-28T19:41:30.947 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 0 2024-07-28T19:41:30.947 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:41:30.959 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (6050 ms) 2024-07-28T19:41:30.959 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-07-28T19:41:30.959 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:30 smithi100 bash[80199]: cluster 2024-07-28T19:41:29.698786+0000 mgr.y (mgr.25358) 255 : cluster [DBG] pgmap v487: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-07-28T19:41:30.996 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:30 smithi100 bash[80199]: cluster 2024-07-28T19:41:29.926996+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-07-28T19:41:30.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:30 smithi100 bash[82644]: cluster 2024-07-28T19:41:29.698786+0000 mgr.y (mgr.25358) 255 : cluster [DBG] pgmap v487: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-07-28T19:41:30.996 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:30 smithi100 bash[82644]: cluster 2024-07-28T19:41:29.926996+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-07-28T19:41:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:30 smithi138 bash[67894]: cluster 2024-07-28T19:41:29.698786+0000 mgr.y (mgr.25358) 255 : cluster [DBG] pgmap v487: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-07-28T19:41:31.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:30 smithi138 bash[67894]: cluster 2024-07-28T19:41:29.926996+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-07-28T19:41:31.971 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 1 2024-07-28T19:41:31.971 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:41:31.971 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:41:31.971 INFO:tasks.workunit.client.0.smithi100.stdout:found snap1 with size 2097152 2024-07-28T19:41:32.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:31 smithi100 bash[80199]: cluster 2024-07-28T19:41:30.923963+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-07-28T19:41:32.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:31 smithi100 bash[82644]: cluster 2024-07-28T19:41:30.923963+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-07-28T19:41:32.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:31 smithi138 bash[67894]: cluster 2024-07-28T19:41:30.923963+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-07-28T19:41:32.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:41:32 smithi138 bash[60615]: ts=2024-07-28T19:41:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:41:32.972 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 2 2024-07-28T19:41:32.972 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:41:32.972 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1 2024-07-28T19:41:32.972 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:41:32.972 INFO:tasks.workunit.client.0.smithi100.stdout:found snap1 with size 2097152 2024-07-28T19:41:32.972 INFO:tasks.workunit.client.0.smithi100.stdout:found snap2 with size 4194304 2024-07-28T19:41:32.977 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 2 2024-07-28T19:41:32.977 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:41:32.977 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap1-rename 2024-07-28T19:41:32.977 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2 2024-07-28T19:41:32.978 INFO:tasks.workunit.client.0.smithi100.stdout:found snap1-rename with size 2097152 2024-07-28T19:41:32.978 INFO:tasks.workunit.client.0.smithi100.stdout:found snap2 with size 4194304 2024-07-28T19:41:33.207 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.208 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.208 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.208 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.209 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.209 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.209 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:32 smithi100 bash[82644]: cluster 2024-07-28T19:41:31.699565+0000 mgr.y (mgr.25358) 256 : cluster [DBG] pgmap v490: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-07-28T19:41:33.209 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:32 smithi100 bash[82644]: cluster 2024-07-28T19:41:31.940504+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-07-28T19:41:33.210 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.210 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:32 smithi100 bash[80199]: cluster 2024-07-28T19:41:31.699565+0000 mgr.y (mgr.25358) 256 : cluster [DBG] pgmap v490: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-07-28T19:41:33.210 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:32 smithi100 bash[80199]: cluster 2024-07-28T19:41:31.940504+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-07-28T19:41:33.210 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.210 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:41:32] "GET /metrics HTTP/1.1" 200 50982 "" "Prometheus/2.43.0" 2024-07-28T19:41:33.210 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:32 smithi138 bash[67894]: cluster 2024-07-28T19:41:31.699565+0000 mgr.y (mgr.25358) 256 : cluster [DBG] pgmap v490: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-07-28T19:41:33.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:32 smithi138 bash[67894]: cluster 2024-07-28T19:41:31.940504+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-07-28T19:41:33.497 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:33 smithi100 systemd[1]: Stopping Ceph osd.2 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:41:33.497 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:33 smithi100 bash[45824]: debug 2024-07-28T19:41:33.321+0000 7efd97e8b640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:41:33.497 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:33 smithi100 bash[45824]: debug 2024-07-28T19:41:33.321+0000 7efd97e8b640 -1 osd.2 614 *** Got signal Terminated *** 2024-07-28T19:41:33.497 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:33 smithi100 bash[45824]: debug 2024-07-28T19:41:33.321+0000 7efd97e8b640 -1 osd.2 614 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:41:34.028 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 1 2024-07-28T19:41:34.028 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:41:34.029 INFO:tasks.workunit.client.0.smithi100.stdout:snap: snap2-rename 2024-07-28T19:41:34.029 INFO:tasks.workunit.client.0.smithi100.stdout:found snap2-rename with size 4194304 2024-07-28T19:41:34.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:33 smithi100 bash[82644]: cluster 2024-07-28T19:41:32.953433+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-07-28T19:41:34.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:33 smithi100 bash[82644]: cluster 2024-07-28T19:41:33.327500+0000 mon.a (mon.0) 806 : cluster [INF] osd.2 marked itself down and dead 2024-07-28T19:41:34.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:33 smithi100 bash[80199]: debug 2024-07-28T19:41:33.941+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:41:34.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:33 smithi100 bash[80199]: cluster 2024-07-28T19:41:32.953433+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-07-28T19:41:34.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:33 smithi100 bash[80199]: cluster 2024-07-28T19:41:33.327500+0000 mon.a (mon.0) 806 : cluster [INF] osd.2 marked itself down and dead 2024-07-28T19:41:34.288 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:41:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:33 smithi138 bash[67894]: cluster 2024-07-28T19:41:32.953433+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-07-28T19:41:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:33 smithi138 bash[67894]: cluster 2024-07-28T19:41:33.327500+0000 mon.a (mon.0) 806 : cluster [INF] osd.2 marked itself down and dead 2024-07-28T19:41:34.946 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 0 2024-07-28T19:41:34.947 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:41:34.951 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (3993 ms) 2024-07-28T19:41:34.952 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-07-28T19:41:35.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[82644]: cluster 2024-07-28T19:41:33.701750+0000 mgr.y (mgr.25358) 257 : cluster [DBG] pgmap v493: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1023 B/s wr, 12 op/s 2024-07-28T19:41:35.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[82644]: cluster 2024-07-28T19:41:33.946353+0000 mon.a (mon.0) 807 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:41:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[82644]: cluster 2024-07-28T19:41:33.946407+0000 mon.a (mon.0) 808 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:41:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[82644]: cluster 2024-07-28T19:41:33.968856+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e616: 8 total, 7 up, 8 in 2024-07-28T19:41:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[82644]: cluster 2024-07-28T19:41:34.881773+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e617: 8 total, 7 up, 8 in 2024-07-28T19:41:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[80199]: cluster 2024-07-28T19:41:33.701750+0000 mgr.y (mgr.25358) 257 : cluster [DBG] pgmap v493: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1023 B/s wr, 12 op/s 2024-07-28T19:41:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[80199]: cluster 2024-07-28T19:41:33.946353+0000 mon.a (mon.0) 807 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:41:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[80199]: cluster 2024-07-28T19:41:33.946407+0000 mon.a (mon.0) 808 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:41:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[80199]: cluster 2024-07-28T19:41:33.968856+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e616: 8 total, 7 up, 8 in 2024-07-28T19:41:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:34 smithi100 bash[80199]: cluster 2024-07-28T19:41:34.881773+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e617: 8 total, 7 up, 8 in 2024-07-28T19:41:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:34 smithi138 bash[67894]: cluster 2024-07-28T19:41:33.701750+0000 mgr.y (mgr.25358) 257 : cluster [DBG] pgmap v493: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1023 B/s wr, 12 op/s 2024-07-28T19:41:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:34 smithi138 bash[67894]: cluster 2024-07-28T19:41:33.946353+0000 mon.a (mon.0) 807 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:41:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:34 smithi138 bash[67894]: cluster 2024-07-28T19:41:33.946407+0000 mon.a (mon.0) 808 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:41:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:34 smithi138 bash[67894]: cluster 2024-07-28T19:41:33.968856+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e616: 8 total, 7 up, 8 in 2024-07-28T19:41:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:34 smithi138 bash[67894]: cluster 2024-07-28T19:41:34.881773+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e617: 8 total, 7 up, 8 in 2024-07-28T19:41:35.746 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:35 smithi100 bash[99250]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-2 2024-07-28T19:41:36.198 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:36 smithi100 bash[80199]: cluster 2024-07-28T19:41:35.956756+0000 mon.a (mon.0) 811 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:41:36.198 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:36 smithi100 bash[80199]: cluster 2024-07-28T19:41:35.956832+0000 mon.a (mon.0) 812 : cluster [WRN] Health check failed: Degraded data redundancy: 11/474 objects degraded (2.321%), 6 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:36.198 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:36 smithi100 bash[80199]: cluster 2024-07-28T19:41:35.974165+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e618: 8 total, 7 up, 8 in 2024-07-28T19:41:36.198 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:36 smithi100 bash[80199]: audit 2024-07-28T19:41:35.980267+0000 mon.a (mon.0) 814 : audit [INF] from='client.? 172.21.15.100:0/4049291410' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:36.199 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:35 smithi100 bash[99345]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-2 2024-07-28T19:41:36.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:36 smithi138 bash[67894]: cluster 2024-07-28T19:41:35.956756+0000 mon.a (mon.0) 811 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:41:36.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:36 smithi138 bash[67894]: cluster 2024-07-28T19:41:35.956832+0000 mon.a (mon.0) 812 : cluster [WRN] Health check failed: Degraded data redundancy: 11/474 objects degraded (2.321%), 6 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:36.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:36 smithi138 bash[67894]: cluster 2024-07-28T19:41:35.974165+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e618: 8 total, 7 up, 8 in 2024-07-28T19:41:36.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:36 smithi138 bash[67894]: audit 2024-07-28T19:41:35.980267+0000 mon.a (mon.0) 814 : audit [INF] from='client.? 172.21.15.100:0/4049291410' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:36.473 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:36 smithi100 bash[82644]: cluster 2024-07-28T19:41:35.956756+0000 mon.a (mon.0) 811 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:41:36.473 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:36 smithi100 bash[82644]: cluster 2024-07-28T19:41:35.956832+0000 mon.a (mon.0) 812 : cluster [WRN] Health check failed: Degraded data redundancy: 11/474 objects degraded (2.321%), 6 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:36.473 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:36 smithi100 bash[82644]: cluster 2024-07-28T19:41:35.974165+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e618: 8 total, 7 up, 8 in 2024-07-28T19:41:36.474 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:36 smithi100 bash[82644]: audit 2024-07-28T19:41:35.980267+0000 mon.a (mon.0) 814 : audit [INF] from='client.? 172.21.15.100:0/4049291410' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:36.982 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:36.983 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:36.983 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:36.983 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:37 smithi138 bash[67894]: cluster 2024-07-28T19:41:35.703117+0000 mgr.y (mgr.25358) 258 : cluster [DBG] pgmap v496: 297 pgs: 9 active+undersized, 19 peering, 20 stale+active+clean, 6 active+undersized+degraded, 243 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.5 KiB/s wr, 87 op/s; 11/474 objects degraded (2.321%) 2024-07-28T19:41:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:37 smithi138 bash[67894]: audit 2024-07-28T19:41:36.976986+0000 mon.a (mon.0) 815 : audit [INF] from='client.? 172.21.15.100:0/4049291410' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:37 smithi138 bash[67894]: cluster 2024-07-28T19:41:36.977467+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e619: 8 total, 7 up, 8 in 2024-07-28T19:41:37.332 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:37 smithi100 bash[80199]: cluster 2024-07-28T19:41:35.703117+0000 mgr.y (mgr.25358) 258 : cluster [DBG] pgmap v496: 297 pgs: 9 active+undersized, 19 peering, 20 stale+active+clean, 6 active+undersized+degraded, 243 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.5 KiB/s wr, 87 op/s; 11/474 objects degraded (2.321%) 2024-07-28T19:41:37.333 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:37 smithi100 bash[80199]: audit 2024-07-28T19:41:36.976986+0000 mon.a (mon.0) 815 : audit [INF] from='client.? 172.21.15.100:0/4049291410' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:37.333 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:37 smithi100 bash[80199]: cluster 2024-07-28T19:41:36.977467+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e619: 8 total, 7 up, 8 in 2024-07-28T19:41:37.333 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:37 smithi100 bash[82644]: cluster 2024-07-28T19:41:35.703117+0000 mgr.y (mgr.25358) 258 : cluster [DBG] pgmap v496: 297 pgs: 9 active+undersized, 19 peering, 20 stale+active+clean, 6 active+undersized+degraded, 243 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 6.5 KiB/s wr, 87 op/s; 11/474 objects degraded (2.321%) 2024-07-28T19:41:37.333 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:37 smithi100 bash[82644]: audit 2024-07-28T19:41:36.976986+0000 mon.a (mon.0) 815 : audit [INF] from='client.? 172.21.15.100:0/4049291410' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:37.333 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:37 smithi100 bash[82644]: cluster 2024-07-28T19:41:36.977467+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e619: 8 total, 7 up, 8 in 2024-07-28T19:41:38.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:38 smithi100 bash[80199]: audit 2024-07-28T19:41:37.641931+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:38.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:38 smithi100 bash[80199]: audit 2024-07-28T19:41:37.644277+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:38.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:38 smithi100 bash[80199]: cluster 2024-07-28T19:41:37.704085+0000 mgr.y (mgr.25358) 259 : cluster [DBG] pgmap v499: 329 pgs: 32 unknown, 9 active+undersized, 19 peering, 20 stale+active+clean, 6 active+undersized+degraded, 243 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 5.5 KiB/s wr, 74 op/s; 11/474 objects degraded (2.321%) 2024-07-28T19:41:38.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:38 smithi100 bash[80199]: cluster 2024-07-28T19:41:37.979564+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e620: 8 total, 7 up, 8 in 2024-07-28T19:41:38.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:38 smithi100 bash[82644]: audit 2024-07-28T19:41:37.641931+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:38.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:38 smithi100 bash[82644]: audit 2024-07-28T19:41:37.644277+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:38.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:38 smithi100 bash[82644]: cluster 2024-07-28T19:41:37.704085+0000 mgr.y (mgr.25358) 259 : cluster [DBG] pgmap v499: 329 pgs: 32 unknown, 9 active+undersized, 19 peering, 20 stale+active+clean, 6 active+undersized+degraded, 243 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 5.5 KiB/s wr, 74 op/s; 11/474 objects degraded (2.321%) 2024-07-28T19:41:38.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:38 smithi100 bash[82644]: cluster 2024-07-28T19:41:37.979564+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e620: 8 total, 7 up, 8 in 2024-07-28T19:41:39.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:38 smithi138 bash[67894]: audit 2024-07-28T19:41:37.641931+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:39.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:38 smithi138 bash[67894]: audit 2024-07-28T19:41:37.644277+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:39.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:38 smithi138 bash[67894]: cluster 2024-07-28T19:41:37.704085+0000 mgr.y (mgr.25358) 259 : cluster [DBG] pgmap v499: 329 pgs: 32 unknown, 9 active+undersized, 19 peering, 20 stale+active+clean, 6 active+undersized+degraded, 243 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 5.5 KiB/s wr, 74 op/s; 11/474 objects degraded (2.321%) 2024-07-28T19:41:39.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:38 smithi138 bash[67894]: cluster 2024-07-28T19:41:37.979564+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e620: 8 total, 7 up, 8 in 2024-07-28T19:41:39.605 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4653 ms) 2024-07-28T19:41:39.605 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-07-28T19:41:40.068 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.069 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.069 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.069 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.069 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.070 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.2.service: Deactivated successfully. 2024-07-28T19:41:40.070 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: Stopped Ceph osd.2 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:41:40.070 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.070 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.070 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.071 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:41:39 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:41:40.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:40 smithi100 bash[80199]: cluster 2024-07-28T19:41:38.982973+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e621: 8 total, 7 up, 8 in 2024-07-28T19:41:40.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:40 smithi100 bash[82644]: cluster 2024-07-28T19:41:38.982973+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e621: 8 total, 7 up, 8 in 2024-07-28T19:41:40.497 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:40 smithi100 systemd[1]: Started Ceph osd.2 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:41:40.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:40 smithi138 bash[67894]: cluster 2024-07-28T19:41:38.982973+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e621: 8 total, 7 up, 8 in 2024-07-28T19:41:41.147 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:41.379 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[73305]: [28/Jul/2024:19:41:41] ENGINE Bus STOPPING 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: cluster 2024-07-28T19:41:39.704957+0000 mgr.y (mgr.25358) 260 : cluster [DBG] pgmap v502: 329 pgs: 8 unknown, 77 active+undersized, 37 active+undersized+degraded, 207 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 4 op/s; 57/477 objects degraded (11.950%) 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: cluster 2024-07-28T19:41:40.096942+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e622: 8 total, 7 up, 8 in 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: audit 2024-07-28T19:41:40.100620+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.100:0/903208924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: audit 2024-07-28T19:41:40.118565+0000 mon.a (mon.0) 822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: audit 2024-07-28T19:41:40.124749+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: audit 2024-07-28T19:41:40.135294+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: audit 2024-07-28T19:41:40.136857+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: cluster 2024-07-28T19:41:40.139848+0000 mon.a (mon.0) 826 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[80199]: audit 2024-07-28T19:41:40.407929+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.100:0/2250817796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: cluster 2024-07-28T19:41:39.704957+0000 mgr.y (mgr.25358) 260 : cluster [DBG] pgmap v502: 329 pgs: 8 unknown, 77 active+undersized, 37 active+undersized+degraded, 207 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 4 op/s; 57/477 objects degraded (11.950%) 2024-07-28T19:41:41.380 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: cluster 2024-07-28T19:41:40.096942+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e622: 8 total, 7 up, 8 in 2024-07-28T19:41:41.381 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: audit 2024-07-28T19:41:40.100620+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.100:0/903208924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:41.381 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: audit 2024-07-28T19:41:40.118565+0000 mon.a (mon.0) 822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:41.381 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: audit 2024-07-28T19:41:40.124749+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:41.381 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: audit 2024-07-28T19:41:40.135294+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:41.381 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: audit 2024-07-28T19:41:40.136857+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:41.381 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: cluster 2024-07-28T19:41:40.139848+0000 mon.a (mon.0) 826 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-07-28T19:41:41.381 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[82644]: audit 2024-07-28T19:41:40.407929+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.100:0/2250817796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: cluster 2024-07-28T19:41:39.704957+0000 mgr.y (mgr.25358) 260 : cluster [DBG] pgmap v502: 329 pgs: 8 unknown, 77 active+undersized, 37 active+undersized+degraded, 207 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 4 op/s; 57/477 objects degraded (11.950%) 2024-07-28T19:41:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: cluster 2024-07-28T19:41:40.096942+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e622: 8 total, 7 up, 8 in 2024-07-28T19:41:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: audit 2024-07-28T19:41:40.100620+0000 mon.c (mon.2) 7 : audit [INF] from='client.? 172.21.15.100:0/903208924' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: audit 2024-07-28T19:41:40.118565+0000 mon.a (mon.0) 822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:41:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: audit 2024-07-28T19:41:40.124749+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: audit 2024-07-28T19:41:40.135294+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: audit 2024-07-28T19:41:40.136857+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: cluster 2024-07-28T19:41:40.139848+0000 mon.a (mon.0) 826 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-07-28T19:41:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:41 smithi138 bash[67894]: audit 2024-07-28T19:41:40.407929+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.100:0/2250817796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:41.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[73305]: [28/Jul/2024:19:41:41] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:41:41.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[73305]: [28/Jul/2024:19:41:41] ENGINE Bus STOPPED 2024-07-28T19:41:41.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[73305]: [28/Jul/2024:19:41:41] ENGINE Bus STARTING 2024-07-28T19:41:41.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[73305]: [28/Jul/2024:19:41:41] ENGINE Serving on http://:::9283 2024-07-28T19:41:41.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[73305]: [28/Jul/2024:19:41:41] ENGINE Bus STARTED 2024-07-28T19:41:42.123 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[99625]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-07-28T19:41:42.123 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[99625]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-07-28T19:41:42.124 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[99625]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-07-28T19:41:42.124 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[99625]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-07-28T19:41:42.124 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[99625]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-07-28T19:41:42.124 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[99625]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-07-28T19:41:42.124 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:41 smithi100 bash[99625]: --> ceph-volume raw activate successful for osd ID: 2 2024-07-28T19:41:42.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:42 smithi100 bash[80199]: audit 2024-07-28T19:41:41.101997+0000 mon.a (mon.0) 827 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:42.496 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:42 smithi100 bash[80199]: cluster 2024-07-28T19:41:41.102240+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2024-07-28T19:41:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:42 smithi100 bash[80199]: audit 2024-07-28T19:41:41.151733+0000 mon.a (mon.0) 829 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:41:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:42 smithi100 bash[82644]: audit 2024-07-28T19:41:41.101997+0000 mon.a (mon.0) 827 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:42 smithi100 bash[82644]: cluster 2024-07-28T19:41:41.102240+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2024-07-28T19:41:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:42 smithi100 bash[82644]: audit 2024-07-28T19:41:41.151733+0000 mon.a (mon.0) 829 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:41:42.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:41:42 smithi138 bash[60615]: ts=2024-07-28T19:41:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:41:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:42 smithi138 bash[67894]: audit 2024-07-28T19:41:41.101997+0000 mon.a (mon.0) 827 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:41:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:42 smithi138 bash[67894]: cluster 2024-07-28T19:41:41.102240+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2024-07-28T19:41:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:42 smithi138 bash[67894]: audit 2024-07-28T19:41:41.151733+0000 mon.a (mon.0) 829 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:41:43.135 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:41:42] "GET /metrics HTTP/1.1" 200 53420 "" "Prometheus/2.43.0" 2024-07-28T19:41:43.325 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:41:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[82644]: cluster 2024-07-28T19:41:41.705668+0000 mgr.y (mgr.25358) 261 : cluster [DBG] pgmap v505: 361 pgs: 32 unknown, 79 active+undersized, 37 active+undersized+degraded, 213 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 57/480 objects degraded (11.875%) 2024-07-28T19:41:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[82644]: cluster 2024-07-28T19:41:42.122805+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2024-07-28T19:41:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[82644]: cluster 2024-07-28T19:41:42.148179+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 57/480 objects degraded (11.875%), 37 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:43.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[82644]: audit 2024-07-28T19:41:42.710092+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.100:0/1703842304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:43.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[80199]: cluster 2024-07-28T19:41:41.705668+0000 mgr.y (mgr.25358) 261 : cluster [DBG] pgmap v505: 361 pgs: 32 unknown, 79 active+undersized, 37 active+undersized+degraded, 213 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 57/480 objects degraded (11.875%) 2024-07-28T19:41:43.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[80199]: cluster 2024-07-28T19:41:42.122805+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2024-07-28T19:41:43.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[80199]: cluster 2024-07-28T19:41:42.148179+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 57/480 objects degraded (11.875%), 37 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:43.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[80199]: audit 2024-07-28T19:41:42.710092+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.100:0/1703842304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:43 smithi138 bash[67894]: cluster 2024-07-28T19:41:41.705668+0000 mgr.y (mgr.25358) 261 : cluster [DBG] pgmap v505: 361 pgs: 32 unknown, 79 active+undersized, 37 active+undersized+degraded, 213 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 1.7 KiB/s wr, 9 op/s; 57/480 objects degraded (11.875%) 2024-07-28T19:41:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:43 smithi138 bash[67894]: cluster 2024-07-28T19:41:42.122805+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2024-07-28T19:41:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:43 smithi138 bash[67894]: cluster 2024-07-28T19:41:42.148179+0000 mon.a (mon.0) 831 : cluster [WRN] Health check update: Degraded data redundancy: 57/480 objects degraded (11.875%), 37 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:43 smithi138 bash[67894]: audit 2024-07-28T19:41:42.710092+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.100:0/1703842304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:43.996 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:43 smithi100 bash[99897]: debug 2024-07-28T19:41:43.793+0000 7fc2fdc80640 1 -- 172.21.15.100:0/2671363384 <== mon.0 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5633a45fd1e0 con 0x5633a45bdc00 2024-07-28T19:41:44.496 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:44 smithi100 bash[99897]: debug 2024-07-28T19:41:44.333+0000 7fc3004e6640 -1 Falling back to public interface 2024-07-28T19:41:44.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:44 smithi100 bash[80199]: cluster 2024-07-28T19:41:43.135478+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2024-07-28T19:41:44.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:44 smithi100 bash[80199]: audit 2024-07-28T19:41:43.351950+0000 mon.a (mon.0) 834 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-07-28T19:41:44.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:44 smithi100 bash[82644]: cluster 2024-07-28T19:41:43.135478+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2024-07-28T19:41:44.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:44 smithi100 bash[82644]: audit 2024-07-28T19:41:43.351950+0000 mon.a (mon.0) 834 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-07-28T19:41:44.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:44 smithi138 bash[67894]: cluster 2024-07-28T19:41:43.135478+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2024-07-28T19:41:44.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:44 smithi138 bash[67894]: audit 2024-07-28T19:41:43.351950+0000 mon.a (mon.0) 834 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-07-28T19:41:45.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:45 smithi100 bash[80199]: cluster 2024-07-28T19:41:43.708604+0000 mgr.y (mgr.25358) 262 : cluster [DBG] pgmap v508: 361 pgs: 10 unknown, 88 active+undersized, 39 active+undersized+degraded, 224 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 152 op/s; 59/498 objects degraded (11.847%) 2024-07-28T19:41:45.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:45 smithi100 bash[80199]: audit 2024-07-28T19:41:44.159265+0000 mon.a (mon.0) 835 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-07-28T19:41:45.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:45 smithi100 bash[80199]: cluster 2024-07-28T19:41:44.159424+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2024-07-28T19:41:45.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:45 smithi100 bash[80199]: audit 2024-07-28T19:41:45.008346+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.100:0/632698193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:45.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:45 smithi100 bash[82644]: cluster 2024-07-28T19:41:43.708604+0000 mgr.y (mgr.25358) 262 : cluster [DBG] pgmap v508: 361 pgs: 10 unknown, 88 active+undersized, 39 active+undersized+degraded, 224 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 152 op/s; 59/498 objects degraded (11.847%) 2024-07-28T19:41:45.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:45 smithi100 bash[82644]: audit 2024-07-28T19:41:44.159265+0000 mon.a (mon.0) 835 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-07-28T19:41:45.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:45 smithi100 bash[82644]: cluster 2024-07-28T19:41:44.159424+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2024-07-28T19:41:45.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:45 smithi100 bash[82644]: audit 2024-07-28T19:41:45.008346+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.100:0/632698193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:45 smithi138 bash[67894]: cluster 2024-07-28T19:41:43.708604+0000 mgr.y (mgr.25358) 262 : cluster [DBG] pgmap v508: 361 pgs: 10 unknown, 88 active+undersized, 39 active+undersized+degraded, 224 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 10 KiB/s wr, 152 op/s; 59/498 objects degraded (11.847%) 2024-07-28T19:41:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:45 smithi138 bash[67894]: audit 2024-07-28T19:41:44.159265+0000 mon.a (mon.0) 835 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-07-28T19:41:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:45 smithi138 bash[67894]: cluster 2024-07-28T19:41:44.159424+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2024-07-28T19:41:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:45 smithi138 bash[67894]: audit 2024-07-28T19:41:45.008346+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.100:0/632698193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:47.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[80199]: cluster 2024-07-28T19:41:45.710381+0000 mgr.y (mgr.25358) 263 : cluster [DBG] pgmap v510: 361 pgs: 92 active+undersized, 39 active+undersized+degraded, 230 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 8.7 KiB/s wr, 154 op/s; 59/501 objects degraded (11.776%) 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[80199]: audit 2024-07-28T19:41:45.926050+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[80199]: audit 2024-07-28T19:41:45.936415+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[80199]: audit 2024-07-28T19:41:46.703957+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[80199]: audit 2024-07-28T19:41:46.710091+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[82644]: cluster 2024-07-28T19:41:45.710381+0000 mgr.y (mgr.25358) 263 : cluster [DBG] pgmap v510: 361 pgs: 92 active+undersized, 39 active+undersized+degraded, 230 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 8.7 KiB/s wr, 154 op/s; 59/501 objects degraded (11.776%) 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[82644]: audit 2024-07-28T19:41:45.926050+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[82644]: audit 2024-07-28T19:41:45.936415+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[82644]: audit 2024-07-28T19:41:46.703957+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:46 smithi100 bash[82644]: audit 2024-07-28T19:41:46.710091+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:46 smithi138 bash[67894]: cluster 2024-07-28T19:41:45.710381+0000 mgr.y (mgr.25358) 263 : cluster [DBG] pgmap v510: 361 pgs: 92 active+undersized, 39 active+undersized+degraded, 230 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 106 KiB/s rd, 8.7 KiB/s wr, 154 op/s; 59/501 objects degraded (11.776%) 2024-07-28T19:41:47.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:46 smithi138 bash[67894]: audit 2024-07-28T19:41:45.926050+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:46 smithi138 bash[67894]: audit 2024-07-28T19:41:45.936415+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:46 smithi138 bash[67894]: audit 2024-07-28T19:41:46.703957+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:46 smithi138 bash[67894]: audit 2024-07-28T19:41:46.710091+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:47.922 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[99897]: debug 2024-07-28T19:41:47.517+0000 7fc3004e6640 -1 osd.2 615 log_to_monitors true 2024-07-28T19:41:48.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[82644]: audit 2024-07-28T19:41:47.291950+0000 mon.a (mon.0) 841 : audit [DBG] from='client.? 172.21.15.100:0/3055631226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:48.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[82644]: audit 2024-07-28T19:41:47.365490+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:48.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[82644]: audit 2024-07-28T19:41:47.372495+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:48.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[82644]: audit 2024-07-28T19:41:47.527435+0000 mon.a (mon.0) 844 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:41:48.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[80199]: audit 2024-07-28T19:41:47.291950+0000 mon.a (mon.0) 841 : audit [DBG] from='client.? 172.21.15.100:0/3055631226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:48.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[80199]: audit 2024-07-28T19:41:47.365490+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:48.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[80199]: audit 2024-07-28T19:41:47.372495+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:48.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:47 smithi100 bash[80199]: audit 2024-07-28T19:41:47.527435+0000 mon.a (mon.0) 844 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:41:48.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:47 smithi138 bash[67894]: audit 2024-07-28T19:41:47.291950+0000 mon.a (mon.0) 841 : audit [DBG] from='client.? 172.21.15.100:0/3055631226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:48.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:47 smithi138 bash[67894]: audit 2024-07-28T19:41:47.365490+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:48.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:47 smithi138 bash[67894]: audit 2024-07-28T19:41:47.372495+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:48.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:47 smithi138 bash[67894]: audit 2024-07-28T19:41:47.527435+0000 mon.a (mon.0) 844 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-07-28T19:41:49.246 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:41:48 smithi100 bash[99897]: debug 2024-07-28T19:41:48.941+0000 7fc2f4288640 -1 osd.2 615 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-28T19:41:49.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[80199]: cluster 2024-07-28T19:41:47.711165+0000 mgr.y (mgr.25358) 264 : cluster [DBG] pgmap v511: 361 pgs: 92 active+undersized, 39 active+undersized+degraded, 230 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 6.7 KiB/s wr, 118 op/s; 59/501 objects degraded (11.776%) 2024-07-28T19:41:49.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[80199]: audit 2024-07-28T19:41:48.388197+0000 mon.a (mon.0) 845 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-28T19:41:49.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[80199]: cluster 2024-07-28T19:41:48.388426+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e627: 8 total, 7 up, 8 in 2024-07-28T19:41:49.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[80199]: audit 2024-07-28T19:41:48.391612+0000 mon.a (mon.0) 847 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:41:49.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[80199]: audit 2024-07-28T19:41:48.915012+0000 mon.a (mon.0) 848 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' 2024-07-28T19:41:49.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[82644]: cluster 2024-07-28T19:41:47.711165+0000 mgr.y (mgr.25358) 264 : cluster [DBG] pgmap v511: 361 pgs: 92 active+undersized, 39 active+undersized+degraded, 230 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 6.7 KiB/s wr, 118 op/s; 59/501 objects degraded (11.776%) 2024-07-28T19:41:49.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[82644]: audit 2024-07-28T19:41:48.388197+0000 mon.a (mon.0) 845 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-28T19:41:49.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[82644]: cluster 2024-07-28T19:41:48.388426+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e627: 8 total, 7 up, 8 in 2024-07-28T19:41:49.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[82644]: audit 2024-07-28T19:41:48.391612+0000 mon.a (mon.0) 847 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:41:49.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[82644]: audit 2024-07-28T19:41:48.915012+0000 mon.a (mon.0) 848 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' 2024-07-28T19:41:49.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:49 smithi138 bash[67894]: cluster 2024-07-28T19:41:47.711165+0000 mgr.y (mgr.25358) 264 : cluster [DBG] pgmap v511: 361 pgs: 92 active+undersized, 39 active+undersized+degraded, 230 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 6.7 KiB/s wr, 118 op/s; 59/501 objects degraded (11.776%) 2024-07-28T19:41:49.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:49 smithi138 bash[67894]: audit 2024-07-28T19:41:48.388197+0000 mon.a (mon.0) 845 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-07-28T19:41:49.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:49 smithi138 bash[67894]: cluster 2024-07-28T19:41:48.388426+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e627: 8 total, 7 up, 8 in 2024-07-28T19:41:49.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:49 smithi138 bash[67894]: audit 2024-07-28T19:41:48.391612+0000 mon.a (mon.0) 847 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:41:49.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:49 smithi138 bash[67894]: audit 2024-07-28T19:41:48.915012+0000 mon.a (mon.0) 848 : audit [INF] from='osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733]' entity='osd.2' 2024-07-28T19:41:50.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:49 smithi100 bash[80199]: debug 2024-07-28T19:41:49.912+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:41:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[80199]: audit 2024-07-28T19:41:49.583191+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.100:0/3864853421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[80199]: cluster 2024-07-28T19:41:49.714433+0000 mgr.y (mgr.25358) 265 : cluster [DBG] pgmap v513: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 785 KiB/s wr, 116 op/s; 61/507 objects degraded (12.032%) 2024-07-28T19:41:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[80199]: cluster 2024-07-28T19:41:49.913908+0000 mon.a (mon.0) 850 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:41:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[80199]: cluster 2024-07-28T19:41:49.913950+0000 mon.a (mon.0) 851 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:41:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[80199]: cluster 2024-07-28T19:41:49.924776+0000 mon.a (mon.0) 852 : cluster [INF] osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733] boot 2024-07-28T19:41:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[80199]: cluster 2024-07-28T19:41:49.924851+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-07-28T19:41:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[80199]: audit 2024-07-28T19:41:49.926357+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[80199]: cluster 2024-07-28T19:41:50.089850+0000 mon.a (mon.0) 855 : cluster [WRN] Health check update: Degraded data redundancy: 59/501 objects degraded (11.776%), 39 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[82644]: audit 2024-07-28T19:41:49.583191+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.100:0/3864853421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[82644]: cluster 2024-07-28T19:41:49.714433+0000 mgr.y (mgr.25358) 265 : cluster [DBG] pgmap v513: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 785 KiB/s wr, 116 op/s; 61/507 objects degraded (12.032%) 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[82644]: cluster 2024-07-28T19:41:49.913908+0000 mon.a (mon.0) 850 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[82644]: cluster 2024-07-28T19:41:49.913950+0000 mon.a (mon.0) 851 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[82644]: cluster 2024-07-28T19:41:49.924776+0000 mon.a (mon.0) 852 : cluster [INF] osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733] boot 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[82644]: cluster 2024-07-28T19:41:49.924851+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[82644]: audit 2024-07-28T19:41:49.926357+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:41:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:50 smithi100 bash[82644]: cluster 2024-07-28T19:41:50.089850+0000 mon.a (mon.0) 855 : cluster [WRN] Health check update: Degraded data redundancy: 59/501 objects degraded (11.776%), 39 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:50 smithi138 bash[67894]: audit 2024-07-28T19:41:49.583191+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.100:0/3864853421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:50 smithi138 bash[67894]: cluster 2024-07-28T19:41:49.714433+0000 mgr.y (mgr.25358) 265 : cluster [DBG] pgmap v513: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 785 KiB/s wr, 116 op/s; 61/507 objects degraded (12.032%) 2024-07-28T19:41:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:50 smithi138 bash[67894]: cluster 2024-07-28T19:41:49.913908+0000 mon.a (mon.0) 850 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:41:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:50 smithi138 bash[67894]: cluster 2024-07-28T19:41:49.913950+0000 mon.a (mon.0) 851 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:41:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:50 smithi138 bash[67894]: cluster 2024-07-28T19:41:49.924776+0000 mon.a (mon.0) 852 : cluster [INF] osd.2 [v2:172.21.15.100:6818/286161733,v1:172.21.15.100:6819/286161733] boot 2024-07-28T19:41:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:50 smithi138 bash[67894]: cluster 2024-07-28T19:41:49.924851+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2024-07-28T19:41:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:50 smithi138 bash[67894]: audit 2024-07-28T19:41:49.926357+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-07-28T19:41:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:50 smithi138 bash[67894]: cluster 2024-07-28T19:41:50.089850+0000 mon.a (mon.0) 855 : cluster [WRN] Health check update: Degraded data redundancy: 59/501 objects degraded (11.776%), 39 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:52.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:51 smithi100 bash[80199]: cluster 2024-07-28T19:41:50.925940+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-07-28T19:41:52.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:51 smithi100 bash[80199]: audit 2024-07-28T19:41:51.882480+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.100:0/3242606094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:52.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:51 smithi100 bash[82644]: cluster 2024-07-28T19:41:50.925940+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-07-28T19:41:52.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:51 smithi100 bash[82644]: audit 2024-07-28T19:41:51.882480+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.100:0/3242606094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:51 smithi138 bash[67894]: cluster 2024-07-28T19:41:50.925940+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-07-28T19:41:52.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:51 smithi138 bash[67894]: audit 2024-07-28T19:41:51.882480+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.100:0/3242606094' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:52.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:41:52 smithi138 bash[60615]: ts=2024-07-28T19:41:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:41:53.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:41:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:41:52] "GET /metrics HTTP/1.1" 200 55744 "" "Prometheus/2.43.0" 2024-07-28T19:41:53.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:52 smithi100 bash[80199]: cluster 2024-07-28T19:41:51.715121+0000 mgr.y (mgr.25358) 266 : cluster [DBG] pgmap v516: 361 pgs: 12 peering, 83 active+undersized, 36 active+undersized+degraded, 230 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 853 KiB/s wr, 9 op/s; 51/507 objects degraded (10.059%) 2024-07-28T19:41:53.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:52 smithi100 bash[80199]: audit 2024-07-28T19:41:52.640306+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:53.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:52 smithi100 bash[80199]: audit 2024-07-28T19:41:52.642530+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:52 smithi100 bash[82644]: cluster 2024-07-28T19:41:51.715121+0000 mgr.y (mgr.25358) 266 : cluster [DBG] pgmap v516: 361 pgs: 12 peering, 83 active+undersized, 36 active+undersized+degraded, 230 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 853 KiB/s wr, 9 op/s; 51/507 objects degraded (10.059%) 2024-07-28T19:41:53.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:52 smithi100 bash[82644]: audit 2024-07-28T19:41:52.640306+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:53.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:52 smithi100 bash[82644]: audit 2024-07-28T19:41:52.642530+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:52 smithi138 bash[67894]: cluster 2024-07-28T19:41:51.715121+0000 mgr.y (mgr.25358) 266 : cluster [DBG] pgmap v516: 361 pgs: 12 peering, 83 active+undersized, 36 active+undersized+degraded, 230 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 853 KiB/s wr, 9 op/s; 51/507 objects degraded (10.059%) 2024-07-28T19:41:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:52 smithi138 bash[67894]: audit 2024-07-28T19:41:52.640306+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:53.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:52 smithi138 bash[67894]: audit 2024-07-28T19:41:52.642530+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:41:55.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:54 smithi100 bash[80199]: cluster 2024-07-28T19:41:53.717834+0000 mgr.y (mgr.25358) 267 : cluster [DBG] pgmap v517: 361 pgs: 12 peering, 15 active+undersized, 4 active+undersized+degraded, 330 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.5 MiB/s wr, 9 op/s; 6/510 objects degraded (1.176%); 1023 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:55.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:54 smithi100 bash[80199]: audit 2024-07-28T19:41:54.166106+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.100:0/1353243848' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:55.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:54 smithi100 bash[82644]: cluster 2024-07-28T19:41:53.717834+0000 mgr.y (mgr.25358) 267 : cluster [DBG] pgmap v517: 361 pgs: 12 peering, 15 active+undersized, 4 active+undersized+degraded, 330 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.5 MiB/s wr, 9 op/s; 6/510 objects degraded (1.176%); 1023 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:55.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:54 smithi100 bash[82644]: audit 2024-07-28T19:41:54.166106+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.100:0/1353243848' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:55.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:54 smithi138 bash[67894]: cluster 2024-07-28T19:41:53.717834+0000 mgr.y (mgr.25358) 267 : cluster [DBG] pgmap v517: 361 pgs: 12 peering, 15 active+undersized, 4 active+undersized+degraded, 330 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.5 MiB/s wr, 9 op/s; 6/510 objects degraded (1.176%); 1023 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:55.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:54 smithi138 bash[67894]: audit 2024-07-28T19:41:54.166106+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.100:0/1353243848' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:56.246 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:55 smithi100 bash[80199]: cluster 2024-07-28T19:41:55.090954+0000 mon.a (mon.0) 859 : cluster [WRN] Health check update: Degraded data redundancy: 6/510 objects degraded (1.176%), 4 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:56.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:55 smithi100 bash[82644]: cluster 2024-07-28T19:41:55.090954+0000 mon.a (mon.0) 859 : cluster [WRN] Health check update: Degraded data redundancy: 6/510 objects degraded (1.176%), 4 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:56.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:55 smithi138 bash[67894]: cluster 2024-07-28T19:41:55.090954+0000 mon.a (mon.0) 859 : cluster [WRN] Health check update: Degraded data redundancy: 6/510 objects degraded (1.176%), 4 pgs degraded (PG_DEGRADED) 2024-07-28T19:41:57.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:56 smithi100 bash[80199]: cluster 2024-07-28T19:41:55.719574+0000 mgr.y (mgr.25358) 268 : cluster [DBG] pgmap v518: 361 pgs: 361 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.2 MiB/s wr, 9 op/s; 838 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:57.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:56 smithi100 bash[80199]: cluster 2024-07-28T19:41:55.935185+0000 mon.a (mon.0) 860 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/510 objects degraded (1.176%), 4 pgs degraded) 2024-07-28T19:41:57.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:56 smithi100 bash[80199]: audit 2024-07-28T19:41:56.454779+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.100:0/787214294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:57.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:56 smithi100 bash[82644]: cluster 2024-07-28T19:41:55.719574+0000 mgr.y (mgr.25358) 268 : cluster [DBG] pgmap v518: 361 pgs: 361 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.2 MiB/s wr, 9 op/s; 838 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:57.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:56 smithi100 bash[82644]: cluster 2024-07-28T19:41:55.935185+0000 mon.a (mon.0) 860 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/510 objects degraded (1.176%), 4 pgs degraded) 2024-07-28T19:41:57.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:56 smithi100 bash[82644]: audit 2024-07-28T19:41:56.454779+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.100:0/787214294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:57.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:56 smithi138 bash[67894]: cluster 2024-07-28T19:41:55.719574+0000 mgr.y (mgr.25358) 268 : cluster [DBG] pgmap v518: 361 pgs: 361 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.2 MiB/s wr, 9 op/s; 838 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:57.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:56 smithi138 bash[67894]: cluster 2024-07-28T19:41:55.935185+0000 mon.a (mon.0) 860 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/510 objects degraded (1.176%), 4 pgs degraded) 2024-07-28T19:41:57.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:56 smithi138 bash[67894]: audit 2024-07-28T19:41:56.454779+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.100:0/787214294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.382347+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.391539+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.393460+0000 mon.a (mon.0) 863 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.394829+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.404369+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.561257+0000 mon.a (mon.0) 866 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.565711+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.568294+0000 mon.a (mon.0) 868 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.570314+0000 mon.a (mon.0) 869 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[82644]: audit 2024-07-28T19:41:57.572724+0000 mon.a (mon.0) 870 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.382347+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.391539+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.393460+0000 mon.a (mon.0) 863 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.394829+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.404369+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.561257+0000 mon.a (mon.0) 866 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.565711+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.568294+0000 mon.a (mon.0) 868 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.570314+0000 mon.a (mon.0) 869 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:58 smithi100 bash[80199]: audit 2024-07-28T19:41:57.572724+0000 mon.a (mon.0) 870 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-28T19:41:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.382347+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.391539+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.393460+0000 mon.a (mon.0) 863 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:41:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.394829+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:41:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.404369+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:41:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.561257+0000 mon.a (mon.0) 866 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:41:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.565711+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.568294+0000 mon.a (mon.0) 868 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.570314+0000 mon.a (mon.0) 869 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:41:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:58 smithi138 bash[67894]: audit 2024-07-28T19:41:57.572724+0000 mon.a (mon.0) 870 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-28T19:41:59.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:59 smithi100 bash[80199]: audit 2024-07-28T19:41:57.573102+0000 mgr.y (mgr.25358) 269 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-28T19:41:59.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:59 smithi100 bash[80199]: cephadm 2024-07-28T19:41:57.576175+0000 mgr.y (mgr.25358) 270 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-07-28T19:41:59.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:59 smithi100 bash[80199]: cluster 2024-07-28T19:41:57.720517+0000 mgr.y (mgr.25358) 271 : cluster [DBG] pgmap v519: 361 pgs: 361 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 512 KiB/s wr, 1 op/s; 767 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:59.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:41:59 smithi100 bash[80199]: audit 2024-07-28T19:41:58.740886+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.100:0/419218165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:59.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:59 smithi100 bash[82644]: audit 2024-07-28T19:41:57.573102+0000 mgr.y (mgr.25358) 269 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-28T19:41:59.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:59 smithi100 bash[82644]: cephadm 2024-07-28T19:41:57.576175+0000 mgr.y (mgr.25358) 270 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-07-28T19:41:59.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:59 smithi100 bash[82644]: cluster 2024-07-28T19:41:57.720517+0000 mgr.y (mgr.25358) 271 : cluster [DBG] pgmap v519: 361 pgs: 361 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 512 KiB/s wr, 1 op/s; 767 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:59.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:41:59 smithi100 bash[82644]: audit 2024-07-28T19:41:58.740886+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.100:0/419218165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:41:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:59 smithi138 bash[67894]: audit 2024-07-28T19:41:57.573102+0000 mgr.y (mgr.25358) 269 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-07-28T19:41:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:59 smithi138 bash[67894]: cephadm 2024-07-28T19:41:57.576175+0000 mgr.y (mgr.25358) 270 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-07-28T19:41:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:59 smithi138 bash[67894]: cluster 2024-07-28T19:41:57.720517+0000 mgr.y (mgr.25358) 271 : cluster [DBG] pgmap v519: 361 pgs: 361 active+clean; 9.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 512 KiB/s wr, 1 op/s; 767 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:41:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:41:59 smithi138 bash[67894]: audit 2024-07-28T19:41:58.740886+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.100:0/419218165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:01.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:01 smithi138 bash[67894]: cluster 2024-07-28T19:41:59.723586+0000 mgr.y (mgr.25358) 272 : cluster [DBG] pgmap v520: 361 pgs: 361 active+clean; 15 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 941 KiB/s wr, 2 op/s; 627 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:01 smithi138 bash[67894]: cephadm 2024-07-28T19:42:00.039866+0000 mgr.y (mgr.25358) 273 : cephadm [INF] Upgrade: Updating osd.3 2024-07-28T19:42:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:01 smithi138 bash[67894]: audit 2024-07-28T19:42:00.050236+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:01 smithi138 bash[67894]: audit 2024-07-28T19:42:00.052059+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:42:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:01 smithi138 bash[67894]: audit 2024-07-28T19:42:00.053645+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:01 smithi138 bash[67894]: cephadm 2024-07-28T19:42:00.058605+0000 mgr.y (mgr.25358) 274 : cephadm [INF] Deploying daemon osd.3 on smithi100 2024-07-28T19:42:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:01 smithi138 bash[67894]: cluster 2024-07-28T19:42:00.098950+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-07-28T19:42:01.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:01 smithi138 bash[67894]: audit 2024-07-28T19:42:01.038337+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.100:0/2141764195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:01.392 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[82644]: cluster 2024-07-28T19:41:59.723586+0000 mgr.y (mgr.25358) 272 : cluster [DBG] pgmap v520: 361 pgs: 361 active+clean; 15 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 941 KiB/s wr, 2 op/s; 627 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:01.392 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[82644]: cephadm 2024-07-28T19:42:00.039866+0000 mgr.y (mgr.25358) 273 : cephadm [INF] Upgrade: Updating osd.3 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[82644]: audit 2024-07-28T19:42:00.050236+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[82644]: audit 2024-07-28T19:42:00.052059+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[82644]: audit 2024-07-28T19:42:00.053645+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[82644]: cephadm 2024-07-28T19:42:00.058605+0000 mgr.y (mgr.25358) 274 : cephadm [INF] Deploying daemon osd.3 on smithi100 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[82644]: cluster 2024-07-28T19:42:00.098950+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[82644]: audit 2024-07-28T19:42:01.038337+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.100:0/2141764195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[80199]: cluster 2024-07-28T19:41:59.723586+0000 mgr.y (mgr.25358) 272 : cluster [DBG] pgmap v520: 361 pgs: 361 active+clean; 15 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 941 KiB/s wr, 2 op/s; 627 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[80199]: cephadm 2024-07-28T19:42:00.039866+0000 mgr.y (mgr.25358) 273 : cephadm [INF] Upgrade: Updating osd.3 2024-07-28T19:42:01.393 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[80199]: audit 2024-07-28T19:42:00.050236+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:01.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[80199]: audit 2024-07-28T19:42:00.052059+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-07-28T19:42:01.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[80199]: audit 2024-07-28T19:42:00.053645+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:01.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[80199]: cephadm 2024-07-28T19:42:00.058605+0000 mgr.y (mgr.25358) 274 : cephadm [INF] Deploying daemon osd.3 on smithi100 2024-07-28T19:42:01.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[80199]: cluster 2024-07-28T19:42:00.098950+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-07-28T19:42:01.394 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:01 smithi100 bash[80199]: audit 2024-07-28T19:42:01.038337+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.100:0/2141764195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:42:02 smithi138 bash[60615]: ts=2024-07-28T19:42:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:42:03.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:42:02] "GET /metrics HTTP/1.1" 200 55744 "" "Prometheus/2.43.0" 2024-07-28T19:42:03.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:03 smithi100 bash[82644]: cluster 2024-07-28T19:42:01.724524+0000 mgr.y (mgr.25358) 275 : cluster [DBG] pgmap v522: 361 pgs: 361 active+clean; 15 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 921 KiB/s wr, 2 op/s; 614 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:03.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:03 smithi100 bash[82644]: audit 2024-07-28T19:42:02.848819+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:03.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:03 smithi100 bash[80199]: cluster 2024-07-28T19:42:01.724524+0000 mgr.y (mgr.25358) 275 : cluster [DBG] pgmap v522: 361 pgs: 361 active+clean; 15 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 921 KiB/s wr, 2 op/s; 614 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:03.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:03 smithi100 bash[80199]: audit 2024-07-28T19:42:02.848819+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:03.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:03 smithi138 bash[67894]: cluster 2024-07-28T19:42:01.724524+0000 mgr.y (mgr.25358) 275 : cluster [DBG] pgmap v522: 361 pgs: 361 active+clean; 15 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 921 KiB/s wr, 2 op/s; 614 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:03.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:03 smithi138 bash[67894]: audit 2024-07-28T19:42:02.848819+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:04.319 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:04 smithi100 bash[80199]: audit 2024-07-28T19:42:03.338283+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.100:0/3457929378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:04.319 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:04 smithi100 bash[82644]: audit 2024-07-28T19:42:03.338283+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.100:0/3457929378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:04.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:04 smithi138 bash[67894]: audit 2024-07-28T19:42:03.338283+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.100:0/3457929378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:04.679 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.679 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.681 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.682 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.682 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: Stopping Ceph osd.3 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:42:04.682 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.682 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.683 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.683 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.683 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:42:04 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:04.997 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:04 smithi100 bash[51332]: debug 2024-07-28T19:42:04.680+0000 7efc171ee640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:42:04.997 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:04 smithi100 bash[51332]: debug 2024-07-28T19:42:04.680+0000 7efc171ee640 -1 osd.3 630 *** Got signal Terminated *** 2024-07-28T19:42:04.997 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:04 smithi100 bash[51332]: debug 2024-07-28T19:42:04.680+0000 7efc171ee640 -1 osd.3 630 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:42:05.496 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:05 smithi100 bash[82644]: cluster 2024-07-28T19:42:03.727197+0000 mgr.y (mgr.25358) 276 : cluster [DBG] pgmap v523: 361 pgs: 361 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1023 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:42:05.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:05 smithi100 bash[82644]: cluster 2024-07-28T19:42:04.683391+0000 mon.a (mon.0) 877 : cluster [INF] osd.3 marked itself down and dead 2024-07-28T19:42:05.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:05 smithi100 bash[80199]: debug 2024-07-28T19:42:05.060+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:05.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:05 smithi100 bash[80199]: cluster 2024-07-28T19:42:03.727197+0000 mgr.y (mgr.25358) 276 : cluster [DBG] pgmap v523: 361 pgs: 361 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1023 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:42:05.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:05 smithi100 bash[80199]: cluster 2024-07-28T19:42:04.683391+0000 mon.a (mon.0) 877 : cluster [INF] osd.3 marked itself down and dead 2024-07-28T19:42:05.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:05 smithi138 bash[67894]: cluster 2024-07-28T19:42:03.727197+0000 mgr.y (mgr.25358) 276 : cluster [DBG] pgmap v523: 361 pgs: 361 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1023 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:42:05.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:05 smithi138 bash[67894]: cluster 2024-07-28T19:42:04.683391+0000 mon.a (mon.0) 877 : cluster [INF] osd.3 marked itself down and dead 2024-07-28T19:42:06.304 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.RemoveFullTry (26698 ms) 2024-07-28T19:42:06.304 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[80199]: cluster 2024-07-28T19:42:05.062537+0000 mon.a (mon.0) 878 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[80199]: cluster 2024-07-28T19:42:05.062590+0000 mon.a (mon.0) 879 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[80199]: cluster 2024-07-28T19:42:05.081575+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e631: 8 total, 7 up, 8 in 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[80199]: cluster 2024-07-28T19:42:05.095482+0000 mon.a (mon.0) 881 : cluster [WRN] pool 'test-librbd-smithi100-68802-22' is full (reached quota's max_bytes: 10 MiB) 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[80199]: cluster 2024-07-28T19:42:05.096109+0000 mon.a (mon.0) 882 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[80199]: cluster 2024-07-28T19:42:05.111662+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e632: 8 total, 7 up, 8 in 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[80199]: audit 2024-07-28T19:42:05.631712+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.100:0/667135615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[82644]: cluster 2024-07-28T19:42:05.062537+0000 mon.a (mon.0) 878 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[82644]: cluster 2024-07-28T19:42:05.062590+0000 mon.a (mon.0) 879 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:06.424 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[82644]: cluster 2024-07-28T19:42:05.081575+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e631: 8 total, 7 up, 8 in 2024-07-28T19:42:06.425 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[82644]: cluster 2024-07-28T19:42:05.095482+0000 mon.a (mon.0) 881 : cluster [WRN] pool 'test-librbd-smithi100-68802-22' is full (reached quota's max_bytes: 10 MiB) 2024-07-28T19:42:06.425 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[82644]: cluster 2024-07-28T19:42:05.096109+0000 mon.a (mon.0) 882 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-07-28T19:42:06.425 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[82644]: cluster 2024-07-28T19:42:05.111662+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e632: 8 total, 7 up, 8 in 2024-07-28T19:42:06.425 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[82644]: audit 2024-07-28T19:42:05.631712+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.100:0/667135615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:06.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:06 smithi138 bash[67894]: cluster 2024-07-28T19:42:05.062537+0000 mon.a (mon.0) 878 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:42:06.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:06 smithi138 bash[67894]: cluster 2024-07-28T19:42:05.062590+0000 mon.a (mon.0) 879 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:06.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:06 smithi138 bash[67894]: cluster 2024-07-28T19:42:05.081575+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e631: 8 total, 7 up, 8 in 2024-07-28T19:42:06.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:06 smithi138 bash[67894]: cluster 2024-07-28T19:42:05.095482+0000 mon.a (mon.0) 881 : cluster [WRN] pool 'test-librbd-smithi100-68802-22' is full (reached quota's max_bytes: 10 MiB) 2024-07-28T19:42:06.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:06 smithi138 bash[67894]: cluster 2024-07-28T19:42:05.096109+0000 mon.a (mon.0) 882 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-07-28T19:42:06.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:06 smithi138 bash[67894]: cluster 2024-07-28T19:42:05.111662+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e632: 8 total, 7 up, 8 in 2024-07-28T19:42:06.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:06 smithi138 bash[67894]: audit 2024-07-28T19:42:05.631712+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.100:0/667135615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:06.747 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[103662]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-3 2024-07-28T19:42:06.747 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:06 smithi100 bash[103768]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-3 2024-07-28T19:42:07.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:07 smithi100 bash[80199]: cluster 2024-07-28T19:42:05.728009+0000 mgr.y (mgr.25358) 277 : cluster [DBG] pgmap v526: 361 pgs: 56 stale+active+clean, 305 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 853 KiB/s wr, 2 op/s 2024-07-28T19:42:07.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:07 smithi100 bash[80199]: cluster 2024-07-28T19:42:06.115238+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e633: 8 total, 7 up, 8 in 2024-07-28T19:42:07.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:07 smithi100 bash[82644]: cluster 2024-07-28T19:42:05.728009+0000 mgr.y (mgr.25358) 277 : cluster [DBG] pgmap v526: 361 pgs: 56 stale+active+clean, 305 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 853 KiB/s wr, 2 op/s 2024-07-28T19:42:07.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:07 smithi100 bash[82644]: cluster 2024-07-28T19:42:06.115238+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e633: 8 total, 7 up, 8 in 2024-07-28T19:42:07.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:07 smithi138 bash[67894]: cluster 2024-07-28T19:42:05.728009+0000 mgr.y (mgr.25358) 277 : cluster [DBG] pgmap v526: 361 pgs: 56 stale+active+clean, 305 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 853 KiB/s wr, 2 op/s 2024-07-28T19:42:07.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:07 smithi138 bash[67894]: cluster 2024-07-28T19:42:06.115238+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e633: 8 total, 7 up, 8 in 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[80199]: cluster 2024-07-28T19:42:07.144683+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e634: 8 total, 7 up, 8 in 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[80199]: audit 2024-07-28T19:42:07.152342+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.100:0/839689116' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[80199]: audit 2024-07-28T19:42:07.155090+0000 mon.a (mon.0) 886 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[80199]: audit 2024-07-28T19:42:07.640649+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[80199]: audit 2024-07-28T19:42:07.643101+0000 mon.a (mon.0) 888 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[80199]: audit 2024-07-28T19:42:07.926406+0000 mon.a (mon.0) 889 : audit [DBG] from='client.? 172.21.15.100:0/2264108531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[82644]: cluster 2024-07-28T19:42:07.144683+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e634: 8 total, 7 up, 8 in 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[82644]: audit 2024-07-28T19:42:07.152342+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.100:0/839689116' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[82644]: audit 2024-07-28T19:42:07.155090+0000 mon.a (mon.0) 886 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[82644]: audit 2024-07-28T19:42:07.640649+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:08.219 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[82644]: audit 2024-07-28T19:42:07.643101+0000 mon.a (mon.0) 888 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:08.220 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:08 smithi100 bash[82644]: audit 2024-07-28T19:42:07.926406+0000 mon.a (mon.0) 889 : audit [DBG] from='client.? 172.21.15.100:0/2264108531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:08 smithi138 bash[67894]: cluster 2024-07-28T19:42:07.144683+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e634: 8 total, 7 up, 8 in 2024-07-28T19:42:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:08 smithi138 bash[67894]: audit 2024-07-28T19:42:07.152342+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.100:0/839689116' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:08 smithi138 bash[67894]: audit 2024-07-28T19:42:07.155090+0000 mon.a (mon.0) 886 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:08 smithi138 bash[67894]: audit 2024-07-28T19:42:07.640649+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:08 smithi138 bash[67894]: audit 2024-07-28T19:42:07.643101+0000 mon.a (mon.0) 888 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:08 smithi138 bash[67894]: audit 2024-07-28T19:42:07.926406+0000 mon.a (mon.0) 889 : audit [DBG] from='client.? 172.21.15.100:0/2264108531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:09.221 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.221 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.221 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:08 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.3.service: Deactivated successfully. 2024-07-28T19:42:09.222 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:08 smithi100 systemd[1]: Stopped Ceph osd.3 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:42:09.222 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.222 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.222 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[82644]: cluster 2024-07-28T19:42:07.728537+0000 mgr.y (mgr.25358) 278 : cluster [DBG] pgmap v529: 393 pgs: 32 unknown, 56 stale+active+clean, 305 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 1 op/s 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[82644]: audit 2024-07-28T19:42:08.147452+0000 mon.a (mon.0) 890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[82644]: cluster 2024-07-28T19:42:08.147763+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e635: 8 total, 7 up, 8 in 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[82644]: cluster 2024-07-28T19:42:08.636223+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[82644]: cluster 2024-07-28T19:42:09.145622+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e636: 8 total, 7 up, 8 in 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[82644]: audit 2024-07-28T19:42:09.150074+0000 mon.a (mon.0) 894 : audit [INF] from='client.? 172.21.15.100:0/1164977381' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[80199]: cluster 2024-07-28T19:42:07.728537+0000 mgr.y (mgr.25358) 278 : cluster [DBG] pgmap v529: 393 pgs: 32 unknown, 56 stale+active+clean, 305 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 1 op/s 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[80199]: audit 2024-07-28T19:42:08.147452+0000 mon.a (mon.0) 890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:42:09.223 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[80199]: cluster 2024-07-28T19:42:08.147763+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e635: 8 total, 7 up, 8 in 2024-07-28T19:42:09.224 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[80199]: cluster 2024-07-28T19:42:08.636223+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:42:09.224 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[80199]: cluster 2024-07-28T19:42:09.145622+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e636: 8 total, 7 up, 8 in 2024-07-28T19:42:09.224 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:09 smithi100 bash[80199]: audit 2024-07-28T19:42:09.150074+0000 mon.a (mon.0) 894 : audit [INF] from='client.? 172.21.15.100:0/1164977381' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:09.224 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.224 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:09.496 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:09 smithi100 systemd[1]: Started Ceph osd.3 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:42:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:09 smithi138 bash[67894]: cluster 2024-07-28T19:42:07.728537+0000 mgr.y (mgr.25358) 278 : cluster [DBG] pgmap v529: 393 pgs: 32 unknown, 56 stale+active+clean, 305 active+clean; 20 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 1 op/s 2024-07-28T19:42:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:09 smithi138 bash[67894]: audit 2024-07-28T19:42:08.147452+0000 mon.a (mon.0) 890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:42:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:09 smithi138 bash[67894]: cluster 2024-07-28T19:42:08.147763+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e635: 8 total, 7 up, 8 in 2024-07-28T19:42:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:09 smithi138 bash[67894]: cluster 2024-07-28T19:42:08.636223+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:42:09.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:09 smithi138 bash[67894]: cluster 2024-07-28T19:42:09.145622+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e636: 8 total, 7 up, 8 in 2024-07-28T19:42:09.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:09 smithi138 bash[67894]: audit 2024-07-28T19:42:09.150074+0000 mon.a (mon.0) 894 : audit [INF] from='client.? 172.21.15.100:0/1164977381' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:42:10.176 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:42:10.487 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[73305]: [28/Jul/2024:19:42:10] ENGINE Bus STOPPING 2024-07-28T19:42:10.487 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[73305]: [28/Jul/2024:19:42:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:42:10.487 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[73305]: [28/Jul/2024:19:42:10] ENGINE Bus STOPPED 2024-07-28T19:42:10.487 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[73305]: [28/Jul/2024:19:42:10] ENGINE Bus STARTING 2024-07-28T19:42:10.487 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[82644]: audit 2024-07-28T19:42:09.253605+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:10.487 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[82644]: audit 2024-07-28T19:42:09.261438+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:10.487 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[82644]: audit 2024-07-28T19:42:09.262469+0000 mon.a (mon.0) 897 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:10.487 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[82644]: audit 2024-07-28T19:42:10.146795+0000 mon.a (mon.0) 898 : audit [INF] from='client.? 172.21.15.100:0/1164977381' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[82644]: cluster 2024-07-28T19:42:10.147319+0000 mon.a (mon.0) 899 : cluster [DBG] osdmap e637: 8 total, 7 up, 8 in 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[82644]: audit 2024-07-28T19:42:10.179640+0000 mon.a (mon.0) 900 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[82644]: audit 2024-07-28T19:42:10.241099+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.100:0/1786168714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[80199]: audit 2024-07-28T19:42:09.253605+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[80199]: audit 2024-07-28T19:42:09.261438+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[80199]: audit 2024-07-28T19:42:09.262469+0000 mon.a (mon.0) 897 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[80199]: audit 2024-07-28T19:42:10.146795+0000 mon.a (mon.0) 898 : audit [INF] from='client.? 172.21.15.100:0/1164977381' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[80199]: cluster 2024-07-28T19:42:10.147319+0000 mon.a (mon.0) 899 : cluster [DBG] osdmap e637: 8 total, 7 up, 8 in 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[80199]: audit 2024-07-28T19:42:10.179640+0000 mon.a (mon.0) 900 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:42:10.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[80199]: audit 2024-07-28T19:42:10.241099+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.100:0/1786168714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:10.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:10 smithi138 bash[67894]: audit 2024-07-28T19:42:09.253605+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:10.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:10 smithi138 bash[67894]: audit 2024-07-28T19:42:09.261438+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:10.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:10 smithi138 bash[67894]: audit 2024-07-28T19:42:09.262469+0000 mon.a (mon.0) 897 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:10.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:10 smithi138 bash[67894]: audit 2024-07-28T19:42:10.146795+0000 mon.a (mon.0) 898 : audit [INF] from='client.? 172.21.15.100:0/1164977381' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:42:10.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:10 smithi138 bash[67894]: cluster 2024-07-28T19:42:10.147319+0000 mon.a (mon.0) 899 : cluster [DBG] osdmap e637: 8 total, 7 up, 8 in 2024-07-28T19:42:10.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:10 smithi138 bash[67894]: audit 2024-07-28T19:42:10.179640+0000 mon.a (mon.0) 900 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:42:10.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:10 smithi138 bash[67894]: audit 2024-07-28T19:42:10.241099+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.100:0/1786168714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:10.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[73305]: [28/Jul/2024:19:42:10] ENGINE Serving on http://:::9283 2024-07-28T19:42:10.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[73305]: [28/Jul/2024:19:42:10] ENGINE Bus STARTED 2024-07-28T19:42:11.247 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[103956]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-07-28T19:42:11.247 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[103956]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-07-28T19:42:11.247 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[103956]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-07-28T19:42:11.247 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[103956]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-07-28T19:42:11.247 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[103956]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-07-28T19:42:11.247 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[103956]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-07-28T19:42:11.247 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:10 smithi100 bash[103956]: --> ceph-volume raw activate successful for osd ID: 3 2024-07-28T19:42:11.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:11 smithi138 bash[67894]: cluster 2024-07-28T19:42:09.729894+0000 mgr.y (mgr.25358) 279 : cluster [DBG] pgmap v532: 425 pgs: 113 active+undersized, 38 unknown, 6 stale+active+clean, 36 active+undersized+degraded, 232 active+clean; 584 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 258 KiB/s wr, 8 op/s; 59/492 objects degraded (11.992%) 2024-07-28T19:42:11.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:11 smithi138 bash[67894]: cluster 2024-07-28T19:42:10.258484+0000 mon.a (mon.0) 901 : cluster [WRN] Health check failed: Degraded data redundancy: 59/492 objects degraded (11.992%), 36 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:11.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:11 smithi138 bash[67894]: cluster 2024-07-28T19:42:11.152916+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e638: 8 total, 7 up, 8 in 2024-07-28T19:42:11.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:11 smithi100 bash[80199]: cluster 2024-07-28T19:42:09.729894+0000 mgr.y (mgr.25358) 279 : cluster [DBG] pgmap v532: 425 pgs: 113 active+undersized, 38 unknown, 6 stale+active+clean, 36 active+undersized+degraded, 232 active+clean; 584 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 258 KiB/s wr, 8 op/s; 59/492 objects degraded (11.992%) 2024-07-28T19:42:11.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:11 smithi100 bash[80199]: cluster 2024-07-28T19:42:10.258484+0000 mon.a (mon.0) 901 : cluster [WRN] Health check failed: Degraded data redundancy: 59/492 objects degraded (11.992%), 36 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:11.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:11 smithi100 bash[80199]: cluster 2024-07-28T19:42:11.152916+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e638: 8 total, 7 up, 8 in 2024-07-28T19:42:11.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:11 smithi100 bash[82644]: cluster 2024-07-28T19:42:09.729894+0000 mgr.y (mgr.25358) 279 : cluster [DBG] pgmap v532: 425 pgs: 113 active+undersized, 38 unknown, 6 stale+active+clean, 36 active+undersized+degraded, 232 active+clean; 584 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 258 KiB/s wr, 8 op/s; 59/492 objects degraded (11.992%) 2024-07-28T19:42:11.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:11 smithi100 bash[82644]: cluster 2024-07-28T19:42:10.258484+0000 mon.a (mon.0) 901 : cluster [WRN] Health check failed: Degraded data redundancy: 59/492 objects degraded (11.992%), 36 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:11.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:11 smithi100 bash[82644]: cluster 2024-07-28T19:42:11.152916+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e638: 8 total, 7 up, 8 in 2024-07-28T19:42:12.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:42:12 smithi138 bash[60615]: ts=2024-07-28T19:42:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:42:12.997 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:42:12] "GET /metrics HTTP/1.1" 200 58317 "" "Prometheus/2.43.0" 2024-07-28T19:42:12.997 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:12 smithi100 bash[104222]: debug 2024-07-28T19:42:12.700+0000 7fd4f4c66640 1 -- 172.21.15.100:0/327404222 <== mon.0 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5609738a31e0 con 0x560973863800 2024-07-28T19:42:13.497 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[104222]: debug 2024-07-28T19:42:13.244+0000 7fd4f74cc640 -1 Falling back to public interface 2024-07-28T19:42:13.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[80199]: cluster 2024-07-28T19:42:11.731377+0000 mgr.y (mgr.25358) 280 : cluster [DBG] pgmap v535: 425 pgs: 5 creating+peering, 129 active+undersized, 27 unknown, 42 active+undersized+degraded, 222 active+clean; 584 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 258 KiB/s wr, 8 op/s; 70/492 objects degraded (14.228%) 2024-07-28T19:42:13.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[80199]: cluster 2024-07-28T19:42:12.152282+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-07-28T19:42:13.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[80199]: audit 2024-07-28T19:42:12.196672+0000 mon.a (mon.0) 904 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-07-28T19:42:13.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[80199]: cluster 2024-07-28T19:42:12.255343+0000 mon.a (mon.0) 905 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:42:13.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[80199]: audit 2024-07-28T19:42:12.532772+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.100:0/84010244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[82644]: cluster 2024-07-28T19:42:11.731377+0000 mgr.y (mgr.25358) 280 : cluster [DBG] pgmap v535: 425 pgs: 5 creating+peering, 129 active+undersized, 27 unknown, 42 active+undersized+degraded, 222 active+clean; 584 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 258 KiB/s wr, 8 op/s; 70/492 objects degraded (14.228%) 2024-07-28T19:42:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[82644]: cluster 2024-07-28T19:42:12.152282+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-07-28T19:42:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[82644]: audit 2024-07-28T19:42:12.196672+0000 mon.a (mon.0) 904 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-07-28T19:42:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[82644]: cluster 2024-07-28T19:42:12.255343+0000 mon.a (mon.0) 905 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:42:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:13 smithi100 bash[82644]: audit 2024-07-28T19:42:12.532772+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.100:0/84010244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:13.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:13 smithi138 bash[67894]: cluster 2024-07-28T19:42:11.731377+0000 mgr.y (mgr.25358) 280 : cluster [DBG] pgmap v535: 425 pgs: 5 creating+peering, 129 active+undersized, 27 unknown, 42 active+undersized+degraded, 222 active+clean; 584 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 258 KiB/s wr, 8 op/s; 70/492 objects degraded (14.228%) 2024-07-28T19:42:13.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:13 smithi138 bash[67894]: cluster 2024-07-28T19:42:12.152282+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-07-28T19:42:13.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:13 smithi138 bash[67894]: audit 2024-07-28T19:42:12.196672+0000 mon.a (mon.0) 904 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-07-28T19:42:13.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:13 smithi138 bash[67894]: cluster 2024-07-28T19:42:12.255343+0000 mon.a (mon.0) 905 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:42:13.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:13 smithi138 bash[67894]: audit 2024-07-28T19:42:12.532772+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.100:0/84010244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:14.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:14 smithi100 bash[80199]: audit 2024-07-28T19:42:13.166800+0000 mon.a (mon.0) 907 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-07-28T19:42:14.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:14 smithi100 bash[80199]: cluster 2024-07-28T19:42:13.166947+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-07-28T19:42:14.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:14 smithi100 bash[82644]: audit 2024-07-28T19:42:13.166800+0000 mon.a (mon.0) 907 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-07-28T19:42:14.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:14 smithi100 bash[82644]: cluster 2024-07-28T19:42:13.166947+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-07-28T19:42:14.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:14 smithi138 bash[67894]: audit 2024-07-28T19:42:13.166800+0000 mon.a (mon.0) 907 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi100-68802-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-07-28T19:42:14.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:14 smithi138 bash[67894]: cluster 2024-07-28T19:42:13.166947+0000 mon.a (mon.0) 908 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-07-28T19:42:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[80199]: cluster 2024-07-28T19:42:13.734608+0000 mgr.y (mgr.25358) 281 : cluster [DBG] pgmap v538: 425 pgs: 5 creating+peering, 140 active+undersized, 4 unknown, 42 active+undersized+degraded, 234 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 257 KiB/s wr, 1 op/s; 71/501 objects degraded (14.172%) 2024-07-28T19:42:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[80199]: audit 2024-07-28T19:42:14.829932+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.100:0/3301645498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[80199]: cluster 2024-07-28T19:42:15.100517+0000 mon.a (mon.0) 910 : cluster [INF] pool 'test-librbd-smithi100-68802-22' no longer out of quota; removing NO_QUOTA flag 2024-07-28T19:42:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[80199]: cluster 2024-07-28T19:42:15.101096+0000 mon.a (mon.0) 911 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-07-28T19:42:15.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[80199]: cluster 2024-07-28T19:42:15.109621+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-07-28T19:42:15.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[82644]: cluster 2024-07-28T19:42:13.734608+0000 mgr.y (mgr.25358) 281 : cluster [DBG] pgmap v538: 425 pgs: 5 creating+peering, 140 active+undersized, 4 unknown, 42 active+undersized+degraded, 234 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 257 KiB/s wr, 1 op/s; 71/501 objects degraded (14.172%) 2024-07-28T19:42:15.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[82644]: audit 2024-07-28T19:42:14.829932+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.100:0/3301645498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:15.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[82644]: cluster 2024-07-28T19:42:15.100517+0000 mon.a (mon.0) 910 : cluster [INF] pool 'test-librbd-smithi100-68802-22' no longer out of quota; removing NO_QUOTA flag 2024-07-28T19:42:15.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[82644]: cluster 2024-07-28T19:42:15.101096+0000 mon.a (mon.0) 911 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-07-28T19:42:15.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:15 smithi100 bash[82644]: cluster 2024-07-28T19:42:15.109621+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-07-28T19:42:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:15 smithi138 bash[67894]: cluster 2024-07-28T19:42:13.734608+0000 mgr.y (mgr.25358) 281 : cluster [DBG] pgmap v538: 425 pgs: 5 creating+peering, 140 active+undersized, 4 unknown, 42 active+undersized+degraded, 234 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 257 KiB/s wr, 1 op/s; 71/501 objects degraded (14.172%) 2024-07-28T19:42:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:15 smithi138 bash[67894]: audit 2024-07-28T19:42:14.829932+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.100:0/3301645498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:15 smithi138 bash[67894]: cluster 2024-07-28T19:42:15.100517+0000 mon.a (mon.0) 910 : cluster [INF] pool 'test-librbd-smithi100-68802-22' no longer out of quota; removing NO_QUOTA flag 2024-07-28T19:42:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:15 smithi138 bash[67894]: cluster 2024-07-28T19:42:15.101096+0000 mon.a (mon.0) 911 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-07-28T19:42:15.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:15 smithi138 bash[67894]: cluster 2024-07-28T19:42:15.109621+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-07-28T19:42:16.747 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[104222]: debug 2024-07-28T19:42:16.424+0000 7fd4f74cc640 -1 osd.3 630 log_to_monitors true 2024-07-28T19:42:17.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:16 smithi138 bash[67894]: cluster 2024-07-28T19:42:15.735478+0000 mgr.y (mgr.25358) 282 : cluster [DBG] pgmap v540: 425 pgs: 139 active+undersized, 44 active+undersized+degraded, 242 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 225 KiB/s wr, 12 op/s; 73/513 objects degraded (14.230%) 2024-07-28T19:42:17.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:16 smithi138 bash[67894]: audit 2024-07-28T19:42:15.810669+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:16 smithi138 bash[67894]: audit 2024-07-28T19:42:15.821161+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:16 smithi138 bash[67894]: cluster 2024-07-28T19:42:16.170568+0000 mon.a (mon.0) 915 : cluster [WRN] Health check update: Degraded data redundancy: 73/513 objects degraded (14.230%), 44 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:16 smithi138 bash[67894]: audit 2024-07-28T19:42:16.433143+0000 mon.a (mon.0) 916 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:42:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:16 smithi138 bash[67894]: audit 2024-07-28T19:42:16.552770+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:16 smithi138 bash[67894]: audit 2024-07-28T19:42:16.558204+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.212 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[80199]: cluster 2024-07-28T19:42:15.735478+0000 mgr.y (mgr.25358) 282 : cluster [DBG] pgmap v540: 425 pgs: 139 active+undersized, 44 active+undersized+degraded, 242 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 225 KiB/s wr, 12 op/s; 73/513 objects degraded (14.230%) 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[80199]: audit 2024-07-28T19:42:15.810669+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[80199]: audit 2024-07-28T19:42:15.821161+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[80199]: cluster 2024-07-28T19:42:16.170568+0000 mon.a (mon.0) 915 : cluster [WRN] Health check update: Degraded data redundancy: 73/513 objects degraded (14.230%), 44 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[80199]: audit 2024-07-28T19:42:16.433143+0000 mon.a (mon.0) 916 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[80199]: audit 2024-07-28T19:42:16.552770+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[80199]: audit 2024-07-28T19:42:16.558204+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[82644]: cluster 2024-07-28T19:42:15.735478+0000 mgr.y (mgr.25358) 282 : cluster [DBG] pgmap v540: 425 pgs: 139 active+undersized, 44 active+undersized+degraded, 242 active+clean; 1.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 225 KiB/s wr, 12 op/s; 73/513 objects degraded (14.230%) 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[82644]: audit 2024-07-28T19:42:15.810669+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[82644]: audit 2024-07-28T19:42:15.821161+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.213 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[82644]: cluster 2024-07-28T19:42:16.170568+0000 mon.a (mon.0) 915 : cluster [WRN] Health check update: Degraded data redundancy: 73/513 objects degraded (14.230%), 44 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:17.214 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[82644]: audit 2024-07-28T19:42:16.433143+0000 mon.a (mon.0) 916 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-07-28T19:42:17.214 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[82644]: audit 2024-07-28T19:42:16.552770+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.214 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:16 smithi100 bash[82644]: audit 2024-07-28T19:42:16.558204+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:17.497 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[104222]: debug 2024-07-28T19:42:17.211+0000 7fd4ef276640 -1 osd.3 630 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-28T19:42:18.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[80199]: audit 2024-07-28T19:42:17.101856+0000 mon.a (mon.0) 919 : audit [DBG] from='client.? 172.21.15.100:0/3933452736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:18.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[80199]: audit 2024-07-28T19:42:17.180754+0000 mon.a (mon.0) 920 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-28T19:42:18.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[80199]: cluster 2024-07-28T19:42:17.181697+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e642: 8 total, 7 up, 8 in 2024-07-28T19:42:18.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[80199]: audit 2024-07-28T19:42:17.189081+0000 mon.a (mon.0) 922 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:42:18.176 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[80199]: debug 2024-07-28T19:42:18.175+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:18.177 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[82644]: audit 2024-07-28T19:42:17.101856+0000 mon.a (mon.0) 919 : audit [DBG] from='client.? 172.21.15.100:0/3933452736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:18.177 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[82644]: audit 2024-07-28T19:42:17.180754+0000 mon.a (mon.0) 920 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-28T19:42:18.177 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[82644]: cluster 2024-07-28T19:42:17.181697+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e642: 8 total, 7 up, 8 in 2024-07-28T19:42:18.177 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:17 smithi100 bash[82644]: audit 2024-07-28T19:42:17.189081+0000 mon.a (mon.0) 922 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:42:18.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:17 smithi138 bash[67894]: audit 2024-07-28T19:42:17.101856+0000 mon.a (mon.0) 919 : audit [DBG] from='client.? 172.21.15.100:0/3933452736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:18.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:17 smithi138 bash[67894]: audit 2024-07-28T19:42:17.180754+0000 mon.a (mon.0) 920 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-07-28T19:42:18.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:17 smithi138 bash[67894]: cluster 2024-07-28T19:42:17.181697+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e642: 8 total, 7 up, 8 in 2024-07-28T19:42:18.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:17 smithi138 bash[67894]: audit 2024-07-28T19:42:17.189081+0000 mon.a (mon.0) 922 : audit [INF] from='osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-07-28T19:42:19.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[80199]: cluster 2024-07-28T19:42:17.736836+0000 mgr.y (mgr.25358) 283 : cluster [DBG] pgmap v542: 425 pgs: 139 active+undersized, 44 active+undersized+degraded, 242 active+clean; 1.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 185 KiB/s wr, 10 op/s; 73/513 objects degraded (14.230%) 2024-07-28T19:42:19.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[80199]: cluster 2024-07-28T19:42:18.178716+0000 mon.a (mon.0) 923 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:19.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[80199]: cluster 2024-07-28T19:42:18.178760+0000 mon.a (mon.0) 924 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:42:19.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[80199]: cluster 2024-07-28T19:42:18.188298+0000 mon.a (mon.0) 925 : cluster [INF] osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967] boot 2024-07-28T19:42:19.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[80199]: cluster 2024-07-28T19:42:18.188800+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-07-28T19:42:19.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[80199]: audit 2024-07-28T19:42:18.194688+0000 mon.a (mon.0) 927 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:42:19.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[82644]: cluster 2024-07-28T19:42:17.736836+0000 mgr.y (mgr.25358) 283 : cluster [DBG] pgmap v542: 425 pgs: 139 active+undersized, 44 active+undersized+degraded, 242 active+clean; 1.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 185 KiB/s wr, 10 op/s; 73/513 objects degraded (14.230%) 2024-07-28T19:42:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[82644]: cluster 2024-07-28T19:42:18.178716+0000 mon.a (mon.0) 923 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[82644]: cluster 2024-07-28T19:42:18.178760+0000 mon.a (mon.0) 924 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:42:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[82644]: cluster 2024-07-28T19:42:18.188298+0000 mon.a (mon.0) 925 : cluster [INF] osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967] boot 2024-07-28T19:42:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[82644]: cluster 2024-07-28T19:42:18.188800+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-07-28T19:42:19.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:18 smithi100 bash[82644]: audit 2024-07-28T19:42:18.194688+0000 mon.a (mon.0) 927 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:42:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:18 smithi138 bash[67894]: cluster 2024-07-28T19:42:17.736836+0000 mgr.y (mgr.25358) 283 : cluster [DBG] pgmap v542: 425 pgs: 139 active+undersized, 44 active+undersized+degraded, 242 active+clean; 1.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 185 KiB/s wr, 10 op/s; 73/513 objects degraded (14.230%) 2024-07-28T19:42:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:18 smithi138 bash[67894]: cluster 2024-07-28T19:42:18.178716+0000 mon.a (mon.0) 923 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:18 smithi138 bash[67894]: cluster 2024-07-28T19:42:18.178760+0000 mon.a (mon.0) 924 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:42:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:18 smithi138 bash[67894]: cluster 2024-07-28T19:42:18.188298+0000 mon.a (mon.0) 925 : cluster [INF] osd.3 [v2:172.21.15.100:6826/728540967,v1:172.21.15.100:6827/728540967] boot 2024-07-28T19:42:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:18 smithi138 bash[67894]: cluster 2024-07-28T19:42:18.188800+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-07-28T19:42:19.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:18 smithi138 bash[67894]: audit 2024-07-28T19:42:18.194688+0000 mon.a (mon.0) 927 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-07-28T19:42:19.467 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable)": 4 2024-07-28T19:42:20.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:20 smithi100 bash[80199]: cluster 2024-07-28T19:42:19.201208+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-07-28T19:42:20.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:20 smithi100 bash[80199]: audit 2024-07-28T19:42:19.457193+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.100:0/2535331305' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:20.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:20 smithi100 bash[82644]: cluster 2024-07-28T19:42:19.201208+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-07-28T19:42:20.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:20 smithi100 bash[82644]: audit 2024-07-28T19:42:19.457193+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.100:0/2535331305' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:20.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:20 smithi138 bash[67894]: cluster 2024-07-28T19:42:19.201208+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-07-28T19:42:20.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:20 smithi138 bash[67894]: audit 2024-07-28T19:42:19.457193+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.100:0/2535331305' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-07-28T19:42:20.670 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch upgrade pause' 2024-07-28T19:42:21.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:21 smithi100 bash[80199]: cluster 2024-07-28T19:42:19.738031+0000 mgr.y (mgr.25358) 284 : cluster [DBG] pgmap v545: 425 pgs: 103 peering, 58 active+undersized, 22 active+undersized+degraded, 242 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 221 KiB/s wr, 0 op/s; 35/516 objects degraded (6.783%) 2024-07-28T19:42:21.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:21 smithi100 bash[82644]: cluster 2024-07-28T19:42:19.738031+0000 mgr.y (mgr.25358) 284 : cluster [DBG] pgmap v545: 425 pgs: 103 peering, 58 active+undersized, 22 active+undersized+degraded, 242 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 221 KiB/s wr, 0 op/s; 35/516 objects degraded (6.783%) 2024-07-28T19:42:21.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:21 smithi138 bash[67894]: cluster 2024-07-28T19:42:19.738031+0000 mgr.y (mgr.25358) 284 : cluster [DBG] pgmap v545: 425 pgs: 103 peering, 58 active+undersized, 22 active+undersized+degraded, 242 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 221 KiB/s wr, 0 op/s; 35/516 objects degraded (6.783%) 2024-07-28T19:42:22.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:22 smithi100 bash[80199]: cluster 2024-07-28T19:42:22.197125+0000 mon.a (mon.0) 930 : cluster [WRN] Health check update: Degraded data redundancy: 24/516 objects degraded (4.651%), 16 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:22.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:22 smithi100 bash[82644]: cluster 2024-07-28T19:42:22.197125+0000 mon.a (mon.0) 930 : cluster [WRN] Health check update: Degraded data redundancy: 24/516 objects degraded (4.651%), 16 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:22.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:22 smithi138 bash[67894]: cluster 2024-07-28T19:42:22.197125+0000 mon.a (mon.0) 930 : cluster [WRN] Health check update: Degraded data redundancy: 24/516 objects degraded (4.651%), 16 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:22.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:42:22 smithi138 bash[60615]: ts=2024-07-28T19:42:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:42:23.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:42:22] "GET /metrics HTTP/1.1" 200 60776 "" "Prometheus/2.43.0" 2024-07-28T19:42:23.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:23 smithi100 bash[82644]: cluster 2024-07-28T19:42:21.739699+0000 mgr.y (mgr.25358) 285 : cluster [DBG] pgmap v546: 425 pgs: 103 peering, 47 active+undersized, 16 active+undersized+degraded, 259 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 171 KiB/s wr, 1 op/s; 24/516 objects degraded (4.651%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:23.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:23 smithi100 bash[82644]: audit 2024-07-28T19:42:22.640985+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:23.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:23 smithi100 bash[82644]: audit 2024-07-28T19:42:22.643046+0000 mon.a (mon.0) 932 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:23.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:23 smithi100 bash[80199]: cluster 2024-07-28T19:42:21.739699+0000 mgr.y (mgr.25358) 285 : cluster [DBG] pgmap v546: 425 pgs: 103 peering, 47 active+undersized, 16 active+undersized+degraded, 259 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 171 KiB/s wr, 1 op/s; 24/516 objects degraded (4.651%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:23.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:23 smithi100 bash[80199]: audit 2024-07-28T19:42:22.640985+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:23.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:23 smithi100 bash[80199]: audit 2024-07-28T19:42:22.643046+0000 mon.a (mon.0) 932 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:23 smithi138 bash[67894]: cluster 2024-07-28T19:42:21.739699+0000 mgr.y (mgr.25358) 285 : cluster [DBG] pgmap v546: 425 pgs: 103 peering, 47 active+undersized, 16 active+undersized+degraded, 259 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 171 KiB/s wr, 1 op/s; 24/516 objects degraded (4.651%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:23 smithi138 bash[67894]: audit 2024-07-28T19:42:22.640985+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:23 smithi138 bash[67894]: audit 2024-07-28T19:42:22.643046+0000 mon.a (mon.0) 932 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:24 smithi138 bash[67894]: cluster 2024-07-28T19:42:24.251150+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/516 objects degraded (4.651%), 16 pgs degraded) 2024-07-28T19:42:24.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:24 smithi100 bash[80199]: cluster 2024-07-28T19:42:24.251150+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/516 objects degraded (4.651%), 16 pgs degraded) 2024-07-28T19:42:24.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:24 smithi100 bash[82644]: cluster 2024-07-28T19:42:24.251150+0000 mon.a (mon.0) 933 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/516 objects degraded (4.651%), 16 pgs degraded) 2024-07-28T19:42:25.235 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:42:25.498 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (19194 ms) 2024-07-28T19:42:25.498 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestIO 2024-07-28T19:42:25.498 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:42:25.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:25 smithi100 bash[80199]: cluster 2024-07-28T19:42:23.742845+0000 mgr.y (mgr.25358) 286 : cluster [DBG] pgmap v547: 425 pgs: 23 peering, 402 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 780 B/s rd, 1.4 MiB/s wr, 2 op/s; 780 KiB/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:42:25.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:25 smithi100 bash[80199]: cluster 2024-07-28T19:42:25.103597+0000 mon.a (mon.0) 934 : cluster [WRN] pool 'test-librbd-smithi100-68802-24' is full (reached quota's max_bytes: 10 MiB) 2024-07-28T19:42:25.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:25 smithi100 bash[80199]: cluster 2024-07-28T19:42:25.104245+0000 mon.a (mon.0) 935 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-07-28T19:42:25.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:25 smithi100 bash[80199]: cluster 2024-07-28T19:42:25.112673+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-07-28T19:42:25.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:25 smithi100 bash[82644]: cluster 2024-07-28T19:42:23.742845+0000 mgr.y (mgr.25358) 286 : cluster [DBG] pgmap v547: 425 pgs: 23 peering, 402 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 780 B/s rd, 1.4 MiB/s wr, 2 op/s; 780 KiB/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:42:25.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:25 smithi100 bash[82644]: cluster 2024-07-28T19:42:25.103597+0000 mon.a (mon.0) 934 : cluster [WRN] pool 'test-librbd-smithi100-68802-24' is full (reached quota's max_bytes: 10 MiB) 2024-07-28T19:42:25.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:25 smithi100 bash[82644]: cluster 2024-07-28T19:42:25.104245+0000 mon.a (mon.0) 935 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-07-28T19:42:25.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:25 smithi100 bash[82644]: cluster 2024-07-28T19:42:25.112673+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-07-28T19:42:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:25 smithi138 bash[67894]: cluster 2024-07-28T19:42:23.742845+0000 mgr.y (mgr.25358) 286 : cluster [DBG] pgmap v547: 425 pgs: 23 peering, 402 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 780 B/s rd, 1.4 MiB/s wr, 2 op/s; 780 KiB/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:42:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:25 smithi138 bash[67894]: cluster 2024-07-28T19:42:25.103597+0000 mon.a (mon.0) 934 : cluster [WRN] pool 'test-librbd-smithi100-68802-24' is full (reached quota's max_bytes: 10 MiB) 2024-07-28T19:42:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:25 smithi138 bash[67894]: cluster 2024-07-28T19:42:25.104245+0000 mon.a (mon.0) 935 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-07-28T19:42:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:25 smithi138 bash[67894]: cluster 2024-07-28T19:42:25.112673+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-07-28T19:42:27.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:27 smithi100 bash[80199]: cluster 2024-07-28T19:42:25.743825+0000 mgr.y (mgr.25358) 287 : cluster [DBG] pgmap v549: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 677 KiB/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:42:27.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:27 smithi100 bash[82644]: cluster 2024-07-28T19:42:25.743825+0000 mgr.y (mgr.25358) 287 : cluster [DBG] pgmap v549: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 677 KiB/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:42:27.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:27 smithi138 bash[67894]: cluster 2024-07-28T19:42:25.743825+0000 mgr.y (mgr.25358) 287 : cluster [DBG] pgmap v549: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 677 KiB/s, 1 keys/s, 1 objects/s recovering 2024-07-28T19:42:29.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: cluster 2024-07-28T19:42:27.745346+0000 mgr.y (mgr.25358) 288 : cluster [DBG] pgmap v550: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 958 KiB/s wr, 4 op/s; 598 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:29.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.672067+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.679554+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.680829+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.681759+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.687697+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.772523+0000 mon.a (mon.0) 942 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.775337+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.776897+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.777972+0000 mon.a (mon.0) 945 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:29 smithi138 bash[67894]: audit 2024-07-28T19:42:28.779842+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-28T19:42:29.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: cluster 2024-07-28T19:42:27.745346+0000 mgr.y (mgr.25358) 288 : cluster [DBG] pgmap v550: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 958 KiB/s wr, 4 op/s; 598 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:29.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.672067+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.679554+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.680829+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.681759+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.687697+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.772523+0000 mon.a (mon.0) 942 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.775337+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.776897+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.777972+0000 mon.a (mon.0) 945 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[80199]: audit 2024-07-28T19:42:28.779842+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-28T19:42:29.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: cluster 2024-07-28T19:42:27.745346+0000 mgr.y (mgr.25358) 288 : cluster [DBG] pgmap v550: 425 pgs: 425 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 958 KiB/s wr, 4 op/s; 598 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.672067+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.679554+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.680829+0000 mon.a (mon.0) 939 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.681759+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.687697+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.772523+0000 mon.a (mon.0) 942 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.775337+0000 mon.a (mon.0) 943 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.776897+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.777972+0000 mon.a (mon.0) 945 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:29 smithi100 bash[82644]: audit 2024-07-28T19:42:28.779842+0000 mon.a (mon.0) 946 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-28T19:42:29.796 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:29.797 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:29.798 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:29.799 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.800 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.801 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.802 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 512~512 = 512 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.803 INFO:tasks.workunit.client.0.smithi100.stdout:aio discard: 1536~512 = 0 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.804 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:42:29.805 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.806 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:42:29.807 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.808 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:29.809 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.810 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:29.811 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:29.812 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:30.278 INFO:teuthology.orchestra.run.smithi100.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:42:30.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:30 smithi138 bash[67894]: audit 2024-07-28T19:42:28.780034+0000 mgr.y (mgr.25358) 289 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-28T19:42:30.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:30 smithi138 bash[67894]: cephadm 2024-07-28T19:42:28.781511+0000 mgr.y (mgr.25358) 290 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-07-28T19:42:30.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:30 smithi138 bash[67894]: cluster 2024-07-28T19:42:30.105685+0000 mon.a (mon.0) 947 : cluster [INF] pool 'test-librbd-smithi100-68802-24' no longer out of quota; removing NO_QUOTA flag 2024-07-28T19:42:30.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:30 smithi138 bash[67894]: cluster 2024-07-28T19:42:30.105939+0000 mon.a (mon.0) 948 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-07-28T19:42:30.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:30 smithi138 bash[67894]: cluster 2024-07-28T19:42:30.111499+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-07-28T19:42:30.665 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:30 smithi138 bash[67894]: audit 2024-07-28T19:42:30.281382+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:30.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[80199]: audit 2024-07-28T19:42:28.780034+0000 mgr.y (mgr.25358) 289 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-28T19:42:30.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[80199]: cephadm 2024-07-28T19:42:28.781511+0000 mgr.y (mgr.25358) 290 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-07-28T19:42:30.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[80199]: cluster 2024-07-28T19:42:30.105685+0000 mon.a (mon.0) 947 : cluster [INF] pool 'test-librbd-smithi100-68802-24' no longer out of quota; removing NO_QUOTA flag 2024-07-28T19:42:30.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[80199]: cluster 2024-07-28T19:42:30.105939+0000 mon.a (mon.0) 948 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-07-28T19:42:30.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[80199]: cluster 2024-07-28T19:42:30.111499+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-07-28T19:42:30.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[80199]: audit 2024-07-28T19:42:30.281382+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:30.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[82644]: audit 2024-07-28T19:42:28.780034+0000 mgr.y (mgr.25358) 289 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-07-28T19:42:30.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[82644]: cephadm 2024-07-28T19:42:28.781511+0000 mgr.y (mgr.25358) 290 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-07-28T19:42:30.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[82644]: cluster 2024-07-28T19:42:30.105685+0000 mon.a (mon.0) 947 : cluster [INF] pool 'test-librbd-smithi100-68802-24' no longer out of quota; removing NO_QUOTA flag 2024-07-28T19:42:30.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[82644]: cluster 2024-07-28T19:42:30.105939+0000 mon.a (mon.0) 948 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-07-28T19:42:30.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[82644]: cluster 2024-07-28T19:42:30.111499+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2024-07-28T19:42:30.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:30 smithi100 bash[82644]: audit 2024-07-28T19:42:30.281382+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:31.157 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch ps' 2024-07-28T19:42:31.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:31 smithi100 bash[80199]: cluster 2024-07-28T19:42:29.748666+0000 mgr.y (mgr.25358) 291 : cluster [DBG] pgmap v551: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 1.0 MiB/s wr, 384 op/s; 512 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:31.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:31 smithi100 bash[80199]: audit 2024-07-28T19:42:30.276577+0000 mgr.y (mgr.25358) 292 : audit [DBG] from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:31.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:31 smithi100 bash[80199]: cephadm 2024-07-28T19:42:30.276936+0000 mgr.y (mgr.25358) 293 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:42:31.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:31 smithi100 bash[82644]: cluster 2024-07-28T19:42:29.748666+0000 mgr.y (mgr.25358) 291 : cluster [DBG] pgmap v551: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 1.0 MiB/s wr, 384 op/s; 512 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:31.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:31 smithi100 bash[82644]: audit 2024-07-28T19:42:30.276577+0000 mgr.y (mgr.25358) 292 : audit [DBG] from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:31.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:31 smithi100 bash[82644]: cephadm 2024-07-28T19:42:30.276936+0000 mgr.y (mgr.25358) 293 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:42:31.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:31 smithi138 bash[67894]: cluster 2024-07-28T19:42:29.748666+0000 mgr.y (mgr.25358) 291 : cluster [DBG] pgmap v551: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 1.0 MiB/s wr, 384 op/s; 512 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:42:31.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:31 smithi138 bash[67894]: audit 2024-07-28T19:42:30.276577+0000 mgr.y (mgr.25358) 292 : audit [DBG] from='client.44405 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:31.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:31 smithi138 bash[67894]: cephadm 2024-07-28T19:42:30.276936+0000 mgr.y (mgr.25358) 293 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:5e39bc7adce78742bcb64d34f81354b8e629f78d4f7582f47b0a1206f1ccbf09 2024-07-28T19:42:32.649 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:42:32 smithi138 bash[60615]: ts=2024-07-28T19:42:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:42:32.997 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:42:32] "GET /metrics HTTP/1.1" 200 60776 "" "Prometheus/2.43.0" 2024-07-28T19:42:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[80199]: cephadm 2024-07-28T19:42:31.638298+0000 mgr.y (mgr.25358) 294 : cephadm [INF] Upgrade: Updating osd.4 2024-07-28T19:42:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[80199]: audit 2024-07-28T19:42:31.648965+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[80199]: audit 2024-07-28T19:42:31.651019+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:42:32.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[80199]: audit 2024-07-28T19:42:31.652532+0000 mon.a (mon.0) 953 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:32.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[80199]: cephadm 2024-07-28T19:42:31.663225+0000 mgr.y (mgr.25358) 295 : cephadm [INF] Deploying daemon osd.4 on smithi138 2024-07-28T19:42:32.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[80199]: cluster 2024-07-28T19:42:31.749740+0000 mgr.y (mgr.25358) 296 : cluster [DBG] pgmap v553: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 276 KiB/s rd, 262 KiB/s wr, 497 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:42:32.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[82644]: cephadm 2024-07-28T19:42:31.638298+0000 mgr.y (mgr.25358) 294 : cephadm [INF] Upgrade: Updating osd.4 2024-07-28T19:42:32.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[82644]: audit 2024-07-28T19:42:31.648965+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:32.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[82644]: audit 2024-07-28T19:42:31.651019+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:42:32.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[82644]: audit 2024-07-28T19:42:31.652532+0000 mon.a (mon.0) 953 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:32.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[82644]: cephadm 2024-07-28T19:42:31.663225+0000 mgr.y (mgr.25358) 295 : cephadm [INF] Deploying daemon osd.4 on smithi138 2024-07-28T19:42:32.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:32 smithi100 bash[82644]: cluster 2024-07-28T19:42:31.749740+0000 mgr.y (mgr.25358) 296 : cluster [DBG] pgmap v553: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 276 KiB/s rd, 262 KiB/s wr, 497 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:42:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:32 smithi138 bash[67894]: cephadm 2024-07-28T19:42:31.638298+0000 mgr.y (mgr.25358) 294 : cephadm [INF] Upgrade: Updating osd.4 2024-07-28T19:42:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:32 smithi138 bash[67894]: audit 2024-07-28T19:42:31.648965+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:32 smithi138 bash[67894]: audit 2024-07-28T19:42:31.651019+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-07-28T19:42:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:32 smithi138 bash[67894]: audit 2024-07-28T19:42:31.652532+0000 mon.a (mon.0) 953 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:42:33.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:32 smithi138 bash[67894]: cephadm 2024-07-28T19:42:31.663225+0000 mgr.y (mgr.25358) 295 : cephadm [INF] Deploying daemon osd.4 on smithi138 2024-07-28T19:42:33.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:32 smithi138 bash[67894]: cluster 2024-07-28T19:42:31.749740+0000 mgr.y (mgr.25358) 296 : cluster [DBG] pgmap v553: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 276 KiB/s rd, 262 KiB/s wr, 497 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:42:33.353 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.353 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.353 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.353 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.353 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.353 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.353 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.353 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.354 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.355 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.356 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.357 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.358 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.359 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.360 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:33.361 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestIO (7854 ms) 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-07-28T19:42:33.362 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:42:34.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:33 smithi100 bash[80199]: audit 2024-07-28T19:42:32.902089+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:34.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:33 smithi100 bash[82644]: audit 2024-07-28T19:42:32.902089+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:34.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:33 smithi138 bash[67894]: audit 2024-07-28T19:42:32.902089+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:35.199 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:34 smithi138 bash[67894]: cluster 2024-07-28T19:42:33.752626+0000 mgr.y (mgr.25358) 297 : cluster [DBG] pgmap v554: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 256 KiB/s rd, 243 KiB/s wr, 460 op/s 2024-07-28T19:42:35.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:34 smithi100 bash[82644]: cluster 2024-07-28T19:42:33.752626+0000 mgr.y (mgr.25358) 297 : cluster [DBG] pgmap v554: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 256 KiB/s rd, 243 KiB/s wr, 460 op/s 2024-07-28T19:42:35.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:34 smithi100 bash[80199]: cluster 2024-07-28T19:42:33.752626+0000 mgr.y (mgr.25358) 297 : cluster [DBG] pgmap v554: 425 pgs: 425 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 256 KiB/s rd, 243 KiB/s wr, 460 op/s 2024-07-28T19:42:35.934 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:42:36.423 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.423 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.423 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.424 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.424 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.424 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.425 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.425 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.425 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: Stopping Ceph osd.4 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:42:36.425 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:42:36 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:36.806 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:36 smithi138 bash[27483]: debug 2024-07-28T19:42:36.422+0000 7f690ac71640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:42:36.806 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:36 smithi138 bash[27483]: debug 2024-07-28T19:42:36.422+0000 7f690ac71640 -1 osd.4 646 *** Got signal Terminated *** 2024-07-28T19:42:36.807 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:36 smithi138 bash[27483]: debug 2024-07-28T19:42:36.422+0000 7f690ac71640 -1 osd.4 646 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:42:37.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:36 smithi100 bash[82644]: cluster 2024-07-28T19:42:35.754535+0000 mgr.y (mgr.25358) 298 : cluster [DBG] pgmap v555: 425 pgs: 425 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 314 KiB/s wr, 864 op/s 2024-07-28T19:42:37.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:36 smithi100 bash[82644]: cluster 2024-07-28T19:42:36.426999+0000 mon.a (mon.0) 955 : cluster [INF] osd.4 marked itself down and dead 2024-07-28T19:42:37.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:36 smithi100 bash[80199]: debug 2024-07-28T19:42:36.907+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:37.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:36 smithi100 bash[80199]: cluster 2024-07-28T19:42:35.754535+0000 mgr.y (mgr.25358) 298 : cluster [DBG] pgmap v555: 425 pgs: 425 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 314 KiB/s wr, 864 op/s 2024-07-28T19:42:37.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:36 smithi100 bash[80199]: cluster 2024-07-28T19:42:36.426999+0000 mon.a (mon.0) 955 : cluster [INF] osd.4 marked itself down and dead 2024-07-28T19:42:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:36 smithi138 bash[67894]: cluster 2024-07-28T19:42:35.754535+0000 mgr.y (mgr.25358) 298 : cluster [DBG] pgmap v555: 425 pgs: 425 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 314 KiB/s wr, 864 op/s 2024-07-28T19:42:37.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:36 smithi138 bash[67894]: cluster 2024-07-28T19:42:36.426999+0000 mon.a (mon.0) 955 : cluster [INF] osd.4 marked itself down and dead 2024-07-28T19:42:38.222 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: cluster 2024-07-28T19:42:36.910512+0000 mon.a (mon.0) 956 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:42:38.223 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: cluster 2024-07-28T19:42:36.910556+0000 mon.a (mon.0) 957 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:38.223 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: cluster 2024-07-28T19:42:36.932719+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e647: 8 total, 7 up, 8 in 2024-07-28T19:42:38.223 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: cluster 2024-07-28T19:42:36.941000+0000 mon.a (mon.0) 959 : cluster [INF] osd.4 marked itself dead as of e647 2024-07-28T19:42:38.223 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.588083+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.223 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.588580+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.6", "id": [7, 0]}]: dispatch 2024-07-28T19:42:38.223 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.589021+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.7", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.223 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.589495+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.a", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.223 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.589893+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.590251+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.590604+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.e", "id": [2, 0]}]: dispatch 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.590959+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.10", "id": [6, 4]}]: dispatch 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.591302+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.13", "id": [2, 4]}]: dispatch 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.591657+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.17", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.641240+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.643277+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.928788+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1", "id": [3, 4]}]': finished 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.929060+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.6", "id": [7, 0]}]': finished 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.929256+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.7", "id": [2, 5]}]': finished 2024-07-28T19:42:38.224 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.929448+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.a", "id": [2, 5]}]': finished 2024-07-28T19:42:38.225 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.929654+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 5]}]': finished 2024-07-28T19:42:38.225 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.929944+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [3, 4]}]': finished 2024-07-28T19:42:38.225 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.930187+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.e", "id": [2, 0]}]': finished 2024-07-28T19:42:38.225 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.930377+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.10", "id": [6, 4]}]': finished 2024-07-28T19:42:38.225 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.930554+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.13", "id": [2, 4]}]': finished 2024-07-28T19:42:38.225 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: audit 2024-07-28T19:42:37.930728+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.17", "id": [3, 4]}]': finished 2024-07-28T19:42:38.225 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:37 smithi138 bash[67894]: cluster 2024-07-28T19:42:37.930896+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e648: 8 total, 7 up, 8 in 2024-07-28T19:42:38.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: cluster 2024-07-28T19:42:36.910512+0000 mon.a (mon.0) 956 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:42:38.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: cluster 2024-07-28T19:42:36.910556+0000 mon.a (mon.0) 957 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:38.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: cluster 2024-07-28T19:42:36.932719+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e647: 8 total, 7 up, 8 in 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: cluster 2024-07-28T19:42:36.941000+0000 mon.a (mon.0) 959 : cluster [INF] osd.4 marked itself dead as of e647 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.588083+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.588580+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.6", "id": [7, 0]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.589021+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.7", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.589495+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.a", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.589893+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.590251+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.590604+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.e", "id": [2, 0]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.590959+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.10", "id": [6, 4]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.591302+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.13", "id": [2, 4]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.591657+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.17", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.641240+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.643277+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.928788+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1", "id": [3, 4]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.929060+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.6", "id": [7, 0]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.929256+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.7", "id": [2, 5]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.929448+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.a", "id": [2, 5]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.929654+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 5]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.929944+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [3, 4]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.930187+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.e", "id": [2, 0]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.930377+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.10", "id": [6, 4]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.930554+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.13", "id": [2, 4]}]': finished 2024-07-28T19:42:38.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: audit 2024-07-28T19:42:37.930728+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.17", "id": [3, 4]}]': finished 2024-07-28T19:42:38.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[82644]: cluster 2024-07-28T19:42:37.930896+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e648: 8 total, 7 up, 8 in 2024-07-28T19:42:38.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: cluster 2024-07-28T19:42:36.910512+0000 mon.a (mon.0) 956 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-07-28T19:42:38.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: cluster 2024-07-28T19:42:36.910556+0000 mon.a (mon.0) 957 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:38.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: cluster 2024-07-28T19:42:36.932719+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e647: 8 total, 7 up, 8 in 2024-07-28T19:42:38.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: cluster 2024-07-28T19:42:36.941000+0000 mon.a (mon.0) 959 : cluster [INF] osd.4 marked itself dead as of e647 2024-07-28T19:42:38.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.588083+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.588580+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.6", "id": [7, 0]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.589021+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.7", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.589495+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.a", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.589893+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 5]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.590251+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.590604+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.e", "id": [2, 0]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.590959+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.10", "id": [6, 4]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.591302+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.13", "id": [2, 4]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.591657+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.17", "id": [3, 4]}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.641240+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.643277+0000 mon.a (mon.0) 971 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.928788+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1", "id": [3, 4]}]': finished 2024-07-28T19:42:38.251 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.929060+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.6", "id": [7, 0]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.929256+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.7", "id": [2, 5]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.929448+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.a", "id": [2, 5]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.929654+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 5]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.929944+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [3, 4]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.930187+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.e", "id": [2, 0]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.930377+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.10", "id": [6, 4]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.930554+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.13", "id": [2, 4]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: audit 2024-07-28T19:42:37.930728+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.17", "id": [3, 4]}]': finished 2024-07-28T19:42:38.252 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:37 smithi100 bash[80199]: cluster 2024-07-28T19:42:37.930896+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e648: 8 total, 7 up, 8 in 2024-07-28T19:42:38.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:38 smithi138 bash[73246]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-4 2024-07-28T19:42:38.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:38 smithi138 bash[73310]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-4 2024-07-28T19:42:39.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:38 smithi100 bash[82644]: cluster 2024-07-28T19:42:37.756039+0000 mgr.y (mgr.25358) 299 : cluster [DBG] pgmap v557: 425 pgs: 28 peering, 43 stale+active+clean, 354 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 283 KiB/s rd, 131 KiB/s wr, 603 op/s 2024-07-28T19:42:39.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:38 smithi100 bash[82644]: cluster 2024-07-28T19:42:38.636738+0000 mon.a (mon.0) 983 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:42:39.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:38 smithi100 bash[82644]: cluster 2024-07-28T19:42:38.927852+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e649: 8 total, 7 up, 8 in 2024-07-28T19:42:39.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:38 smithi100 bash[80199]: cluster 2024-07-28T19:42:37.756039+0000 mgr.y (mgr.25358) 299 : cluster [DBG] pgmap v557: 425 pgs: 28 peering, 43 stale+active+clean, 354 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 283 KiB/s rd, 131 KiB/s wr, 603 op/s 2024-07-28T19:42:39.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:38 smithi100 bash[80199]: cluster 2024-07-28T19:42:38.636738+0000 mon.a (mon.0) 983 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:42:39.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:38 smithi100 bash[80199]: cluster 2024-07-28T19:42:38.927852+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e649: 8 total, 7 up, 8 in 2024-07-28T19:42:39.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:38 smithi138 bash[67894]: cluster 2024-07-28T19:42:37.756039+0000 mgr.y (mgr.25358) 299 : cluster [DBG] pgmap v557: 425 pgs: 28 peering, 43 stale+active+clean, 354 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 283 KiB/s rd, 131 KiB/s wr, 603 op/s 2024-07-28T19:42:39.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:38 smithi138 bash[67894]: cluster 2024-07-28T19:42:38.636738+0000 mon.a (mon.0) 983 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-07-28T19:42:39.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:38 smithi138 bash[67894]: cluster 2024-07-28T19:42:38.927852+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e649: 8 total, 7 up, 8 in 2024-07-28T19:42:40.089 INFO:teuthology.orchestra.run.smithi100.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-28T19:42:40.089 INFO:teuthology.orchestra.run.smithi100.stdout:alertmanager.a smithi100 *:9093,9094 running (8m) 24s ago 15m 13.9M - 0.25.0 c8568f914cd2 61edf7ffcf9d 2024-07-28T19:42:40.089 INFO:teuthology.orchestra.run.smithi100.stdout:grafana.a smithi138 *:3000 running (8m) 54s ago 15m 75.0M - 9.4.7 954c08fa6188 fe0244c1bc8a 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi100.ppcqpk smithi100 running (13m) 24s ago 13m 14.9M - 17.2.7-1155-g3120d060 5ec745195f7e 24cab9a6a84c 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi138.nphkis smithi138 running (13m) 54s ago 13m 19.4M - 17.2.7-1155-g3120d060 5ec745195f7e cc36fa0642f5 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.x smithi138 *:8443,9283,8765 running (7m) 54s ago 23m 425M - 18.2.2-1765-g1c7dd747 dd66d6cc220c 6027bd7b0eab 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.y smithi100 *:8443,9283,8765 running (8m) 24s ago 26m 493M - 18.2.2-1765-g1c7dd747 dd66d6cc220c d4cdb32af700 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:mon.a smithi100 running (7m) 24s ago 26m 68.5M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 82b65d214966 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:mon.b smithi138 running (4m) 54s ago 24m 46.2M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 7d6318daf479 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:mon.c smithi100 running (5m) 24s ago 24m 54.5M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 7a8870df1475 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.a smithi100 *:9100 running (8m) 24s ago 16m 8695k - 1.5.0 0da6a335fe13 558573030b79 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.b smithi138 *:9100 running (8m) 54s ago 16m 8536k - 1.5.0 0da6a335fe13 3867a7654990 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:osd.0 smithi100 running (3m) 24s ago 23m 83.6M 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c a2819bfee948 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:osd.1 smithi100 running (2m) 24s ago 22m 89.9M 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c 6ccc7d050acc 2024-07-28T19:42:40.090 INFO:teuthology.orchestra.run.smithi100.stdout:osd.2 smithi100 running (57s) 24s ago 21m 90.0M 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c 9f221b6ac172 2024-07-28T19:42:40.091 INFO:teuthology.orchestra.run.smithi100.stdout:osd.3 smithi100 running (28s) 24s ago 20m - 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c 72d1f6db89df 2024-07-28T19:42:40.091 INFO:teuthology.orchestra.run.smithi100.stdout:osd.4 smithi138 running (19m) 54s ago 19m 127M 2524M 17.2.7-1155-g3120d060 5ec745195f7e feff0c636bab 2024-07-28T19:42:40.091 INFO:teuthology.orchestra.run.smithi100.stdout:osd.5 smithi138 running (18m) 54s ago 18m 120M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 33bea4a7c9ec 2024-07-28T19:42:40.091 INFO:teuthology.orchestra.run.smithi100.stdout:osd.6 smithi138 running (18m) 54s ago 18m 136M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 11e0eaf60c0b 2024-07-28T19:42:40.091 INFO:teuthology.orchestra.run.smithi100.stdout:osd.7 smithi138 running (17m) 54s ago 17m 154M 2524M 17.2.7-1155-g3120d060 5ec745195f7e fd79c6c44f0f 2024-07-28T19:42:40.091 INFO:teuthology.orchestra.run.smithi100.stdout:prometheus.a smithi138 *:9095 running (7m) 54s ago 16m 58.5M - 2.43.0 a07b618ecd1d bc318a7bc21e 2024-07-28T19:42:40.817 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph orch ps' 2024-07-28T19:42:41.192 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.192 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.193 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.193 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.193 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.194 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:40 smithi138 bash[67894]: cluster 2024-07-28T19:42:39.757309+0000 mgr.y (mgr.25358) 300 : cluster [DBG] pgmap v560: 425 pgs: 3 remapped+peering, 61 active+undersized, 31 peering, 11 stale+active+clean, 36 active+undersized+degraded, 283 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 466 KiB/s rd, 919 KiB/s wr, 889 op/s; 54/531 objects degraded (10.169%) 2024-07-28T19:42:41.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:40 smithi138 bash[67894]: cluster 2024-07-28T19:42:39.925873+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e650: 8 total, 7 up, 8 in 2024-07-28T19:42:41.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:40 smithi138 bash[67894]: cluster 2024-07-28T19:42:39.969792+0000 mon.a (mon.0) 986 : cluster [WRN] Health check failed: Degraded data redundancy: 54/531 objects degraded (10.169%), 36 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:41.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:40 smithi138 bash[67894]: audit 2024-07-28T19:42:40.078083+0000 mgr.y (mgr.25358) 301 : audit [DBG] from='client.44411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:41.194 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.195 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.195 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:40 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.4.service: Deactivated successfully. 2024-07-28T19:42:41.195 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:40 smithi138 systemd[1]: Stopped Ceph osd.4 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:42:41.195 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:42:41.195 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:41 smithi138 systemd[1]: Started Ceph osd.4 for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:42:41.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:40 smithi100 bash[80199]: cluster 2024-07-28T19:42:39.757309+0000 mgr.y (mgr.25358) 300 : cluster [DBG] pgmap v560: 425 pgs: 3 remapped+peering, 61 active+undersized, 31 peering, 11 stale+active+clean, 36 active+undersized+degraded, 283 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 466 KiB/s rd, 919 KiB/s wr, 889 op/s; 54/531 objects degraded (10.169%) 2024-07-28T19:42:41.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:40 smithi100 bash[80199]: cluster 2024-07-28T19:42:39.925873+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e650: 8 total, 7 up, 8 in 2024-07-28T19:42:41.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:40 smithi100 bash[80199]: cluster 2024-07-28T19:42:39.969792+0000 mon.a (mon.0) 986 : cluster [WRN] Health check failed: Degraded data redundancy: 54/531 objects degraded (10.169%), 36 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:41.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:40 smithi100 bash[80199]: audit 2024-07-28T19:42:40.078083+0000 mgr.y (mgr.25358) 301 : audit [DBG] from='client.44411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:41.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:40 smithi100 bash[82644]: cluster 2024-07-28T19:42:39.757309+0000 mgr.y (mgr.25358) 300 : cluster [DBG] pgmap v560: 425 pgs: 3 remapped+peering, 61 active+undersized, 31 peering, 11 stale+active+clean, 36 active+undersized+degraded, 283 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 466 KiB/s rd, 919 KiB/s wr, 889 op/s; 54/531 objects degraded (10.169%) 2024-07-28T19:42:41.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:40 smithi100 bash[82644]: cluster 2024-07-28T19:42:39.925873+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e650: 8 total, 7 up, 8 in 2024-07-28T19:42:41.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:40 smithi100 bash[82644]: cluster 2024-07-28T19:42:39.969792+0000 mon.a (mon.0) 986 : cluster [WRN] Health check failed: Degraded data redundancy: 54/531 objects degraded (10.169%), 36 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:41.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:40 smithi100 bash[82644]: audit 2024-07-28T19:42:40.078083+0000 mgr.y (mgr.25358) 301 : audit [DBG] from='client.44411 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:42.341 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:42.341 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:42.341 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:42.341 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.342 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:42:42.343 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.344 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.345 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.346 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:42.347 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 512~512 = 512 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:aio discard: 1536~512 = 0 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:42:42.348 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.349 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.350 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:42:42.351 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.352 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:42.353 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.354 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:42.355 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:42.356 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:42.357 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:42.357 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:42.357 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:42.357 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:42.357 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:42 smithi100 bash[82644]: audit 2024-07-28T19:42:41.237948+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:42 smithi100 bash[82644]: audit 2024-07-28T19:42:41.251728+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:42 smithi100 bash[82644]: audit 2024-07-28T19:42:41.254495+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:42 smithi100 bash[80199]: audit 2024-07-28T19:42:41.237948+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:42 smithi100 bash[80199]: audit 2024-07-28T19:42:41.251728+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:42.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:42 smithi100 bash[80199]: audit 2024-07-28T19:42:41.254495+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[67894]: audit 2024-07-28T19:42:41.237948+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[67894]: audit 2024-07-28T19:42:41.251728+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[67894]: audit 2024-07-28T19:42:41.254495+0000 mon.a (mon.0) 989 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:42:42.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[60615]: ts=2024-07-28T19:42:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:42:43.056 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[73460]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-07-28T19:42:43.056 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[73460]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-07-28T19:42:43.056 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[73460]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-07-28T19:42:43.056 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[73460]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-07-28T19:42:43.057 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[73460]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-07-28T19:42:43.057 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[73460]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-07-28T19:42:43.057 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:42 smithi138 bash[73460]: --> ceph-volume raw activate successful for osd ID: 4 2024-07-28T19:42:43.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:42:42] "GET /metrics HTTP/1.1" 200 60723 "" "Prometheus/2.43.0" 2024-07-28T19:42:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:43 smithi138 bash[67894]: cluster 2024-07-28T19:42:41.759336+0000 mgr.y (mgr.25358) 302 : cluster [DBG] pgmap v562: 425 pgs: 1 activating+remapped, 3 remapped+peering, 84 active+undersized, 31 peering, 44 active+undersized+degraded, 262 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 925 KiB/s wr, 128 op/s; 65/531 objects degraded (12.241%) 2024-07-28T19:42:43.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:43 smithi100 bash[82644]: cluster 2024-07-28T19:42:41.759336+0000 mgr.y (mgr.25358) 302 : cluster [DBG] pgmap v562: 425 pgs: 1 activating+remapped, 3 remapped+peering, 84 active+undersized, 31 peering, 44 active+undersized+degraded, 262 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 925 KiB/s wr, 128 op/s; 65/531 objects degraded (12.241%) 2024-07-28T19:42:43.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:43 smithi100 bash[80199]: cluster 2024-07-28T19:42:41.759336+0000 mgr.y (mgr.25358) 302 : cluster [DBG] pgmap v562: 425 pgs: 1 activating+remapped, 3 remapped+peering, 84 active+undersized, 31 peering, 44 active+undersized+degraded, 262 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 925 KiB/s wr, 128 op/s; 65/531 objects degraded (12.241%) 2024-07-28T19:42:44.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:44 smithi100 bash[80199]: cluster 2024-07-28T19:42:44.256373+0000 mon.a (mon.0) 990 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-07-28T19:42:44.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:44 smithi100 bash[82644]: cluster 2024-07-28T19:42:44.256373+0000 mon.a (mon.0) 990 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-07-28T19:42:44.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:44 smithi138 bash[67894]: cluster 2024-07-28T19:42:44.256373+0000 mon.a (mon.0) 990 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-07-28T19:42:45.306 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:44 smithi138 bash[73689]: debug 2024-07-28T19:42:44.910+0000 7f5fd3741640 1 -- 172.21.15.138:0/1957581877 <== mon.1 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d4a6d511e0 con 0x55d4a6d11800 2024-07-28T19:42:45.618 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:42:45.713 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:45 smithi100 bash[80199]: cluster 2024-07-28T19:42:43.763044+0000 mgr.y (mgr.25358) 303 : cluster [DBG] pgmap v563: 425 pgs: 4 active+clean+remapped, 1 activating+remapped, 3 remapped+peering, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 925 KiB/s rd, 1.1 MiB/s wr, 934 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:45.713 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:45 smithi100 bash[80199]: cluster 2024-07-28T19:42:45.108439+0000 mon.a (mon.0) 991 : cluster [WRN] Health check update: Degraded data redundancy: 76/531 objects degraded (14.313%), 53 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:45.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:45 smithi100 bash[82644]: cluster 2024-07-28T19:42:43.763044+0000 mgr.y (mgr.25358) 303 : cluster [DBG] pgmap v563: 425 pgs: 4 active+clean+remapped, 1 activating+remapped, 3 remapped+peering, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 925 KiB/s rd, 1.1 MiB/s wr, 934 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:45.713 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:45 smithi100 bash[82644]: cluster 2024-07-28T19:42:45.108439+0000 mon.a (mon.0) 991 : cluster [WRN] Health check update: Degraded data redundancy: 76/531 objects degraded (14.313%), 53 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:45.806 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:45 smithi138 bash[73689]: debug 2024-07-28T19:42:45.450+0000 7f5fd5fa7640 -1 Falling back to public interface 2024-07-28T19:42:45.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:45 smithi138 bash[67894]: cluster 2024-07-28T19:42:43.763044+0000 mgr.y (mgr.25358) 303 : cluster [DBG] pgmap v563: 425 pgs: 4 active+clean+remapped, 1 activating+remapped, 3 remapped+peering, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 925 KiB/s rd, 1.1 MiB/s wr, 934 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:45.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:45 smithi138 bash[67894]: cluster 2024-07-28T19:42:45.108439+0000 mon.a (mon.0) 991 : cluster [WRN] Health check update: Degraded data redundancy: 76/531 objects degraded (14.313%), 53 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:47.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:47 smithi100 bash[82644]: cluster 2024-07-28T19:42:45.765124+0000 mgr.y (mgr.25358) 304 : cluster [DBG] pgmap v564: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 708 KiB/s rd, 897 KiB/s wr, 715 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:47.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:47 smithi100 bash[80199]: cluster 2024-07-28T19:42:45.765124+0000 mgr.y (mgr.25358) 304 : cluster [DBG] pgmap v564: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 708 KiB/s rd, 897 KiB/s wr, 715 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:47 smithi138 bash[67894]: cluster 2024-07-28T19:42:45.765124+0000 mgr.y (mgr.25358) 304 : cluster [DBG] pgmap v564: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 708 KiB/s rd, 897 KiB/s wr, 715 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:48.944 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.944 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.944 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.944 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.944 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.944 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.944 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.945 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.946 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.947 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.948 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.949 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.950 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.951 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:48.952 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 4 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:read: 4 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (15591 ms) 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-07-28T19:42:48.953 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:42:49.056 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:48 smithi138 bash[73689]: debug 2024-07-28T19:42:48.697+0000 7f5fd5fa7640 -1 osd.4 647 log_to_monitors true 2024-07-28T19:42:49.171 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[80199]: cluster 2024-07-28T19:42:47.767191+0000 mgr.y (mgr.25358) 305 : cluster [DBG] pgmap v565: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 555 KiB/s rd, 290 KiB/s wr, 558 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:49.172 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[80199]: audit 2024-07-28T19:42:48.176902+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.172 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[80199]: audit 2024-07-28T19:42:48.186948+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.172 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[80199]: audit 2024-07-28T19:42:48.707527+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:42:49.172 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[80199]: audit 2024-07-28T19:42:48.708232+0000 mon.a (mon.0) 994 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:42:49.172 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[80199]: audit 2024-07-28T19:42:48.909198+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.172 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[80199]: audit 2024-07-28T19:42:48.913877+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.208 INFO:teuthology.orchestra.run.smithi100.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-07-28T19:42:49.208 INFO:teuthology.orchestra.run.smithi100.stdout:alertmanager.a smithi100 *:9093,9094 running (8m) 33s ago 15m 13.9M - 0.25.0 c8568f914cd2 61edf7ffcf9d 2024-07-28T19:42:49.208 INFO:teuthology.orchestra.run.smithi100.stdout:grafana.a smithi138 *:3000 running (8m) 1s ago 15m 75.3M - 9.4.7 954c08fa6188 fe0244c1bc8a 2024-07-28T19:42:49.208 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi100.ppcqpk smithi100 running (13m) 33s ago 13m 14.9M - 17.2.7-1155-g3120d060 5ec745195f7e 24cab9a6a84c 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:mds.foo.smithi138.nphkis smithi138 running (13m) 1s ago 13m 19.7M - 17.2.7-1155-g3120d060 5ec745195f7e cc36fa0642f5 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.x smithi138 *:8443,9283,8765 running (7m) 1s ago 24m 426M - 18.2.2-1765-g1c7dd747 dd66d6cc220c 6027bd7b0eab 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:mgr.y smithi100 *:8443,9283,8765 running (8m) 33s ago 26m 493M - 18.2.2-1765-g1c7dd747 dd66d6cc220c d4cdb32af700 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:mon.a smithi100 running (7m) 33s ago 26m 68.5M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 82b65d214966 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:mon.b smithi138 running (5m) 1s ago 24m 50.5M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 7d6318daf479 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:mon.c smithi100 running (5m) 33s ago 24m 54.5M 2048M 18.2.2-1765-g1c7dd747 dd66d6cc220c 7a8870df1475 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.a smithi100 *:9100 running (8m) 33s ago 16m 8695k - 1.5.0 0da6a335fe13 558573030b79 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:node-exporter.b smithi138 *:9100 running (8m) 1s ago 16m 8584k - 1.5.0 0da6a335fe13 3867a7654990 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:osd.0 smithi100 running (3m) 33s ago 23m 83.6M 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c a2819bfee948 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:osd.1 smithi100 running (2m) 33s ago 22m 89.9M 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c 6ccc7d050acc 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:osd.2 smithi100 running (66s) 33s ago 21m 90.0M 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c 9f221b6ac172 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:osd.3 smithi100 running (37s) 33s ago 20m - 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c 72d1f6db89df 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:osd.4 smithi138 running (5s) 1s ago 19m - 2524M 18.2.2-1765-g1c7dd747 dd66d6cc220c aeac134ab2e5 2024-07-28T19:42:49.209 INFO:teuthology.orchestra.run.smithi100.stdout:osd.5 smithi138 running (19m) 1s ago 19m 129M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 33bea4a7c9ec 2024-07-28T19:42:49.210 INFO:teuthology.orchestra.run.smithi100.stdout:osd.6 smithi138 running (18m) 1s ago 18m 143M 2524M 17.2.7-1155-g3120d060 5ec745195f7e 11e0eaf60c0b 2024-07-28T19:42:49.210 INFO:teuthology.orchestra.run.smithi100.stdout:osd.7 smithi138 running (17m) 1s ago 17m 162M 2524M 17.2.7-1155-g3120d060 5ec745195f7e fd79c6c44f0f 2024-07-28T19:42:49.210 INFO:teuthology.orchestra.run.smithi100.stdout:prometheus.a smithi138 *:9095 running (7m) 1s ago 16m 61.7M - 2.43.0 a07b618ecd1d bc318a7bc21e 2024-07-28T19:42:49.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[82644]: cluster 2024-07-28T19:42:47.767191+0000 mgr.y (mgr.25358) 305 : cluster [DBG] pgmap v565: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 555 KiB/s rd, 290 KiB/s wr, 558 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:49.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[82644]: audit 2024-07-28T19:42:48.176902+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[82644]: audit 2024-07-28T19:42:48.186948+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[82644]: audit 2024-07-28T19:42:48.707527+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:42:49.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[82644]: audit 2024-07-28T19:42:48.708232+0000 mon.a (mon.0) 994 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:42:49.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[82644]: audit 2024-07-28T19:42:48.909198+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:49 smithi100 bash[82644]: audit 2024-07-28T19:42:48.913877+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:42:49 smithi138 bash[73689]: debug 2024-07-28T19:42:49.361+0000 7f5fcdd51640 -1 osd.4 647 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-07-28T19:42:49.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:49 smithi138 bash[67894]: cluster 2024-07-28T19:42:47.767191+0000 mgr.y (mgr.25358) 305 : cluster [DBG] pgmap v565: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 555 KiB/s rd, 290 KiB/s wr, 558 op/s; 76/531 objects degraded (14.313%) 2024-07-28T19:42:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:49 smithi138 bash[67894]: audit 2024-07-28T19:42:48.176902+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:49 smithi138 bash[67894]: audit 2024-07-28T19:42:48.186948+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:49 smithi138 bash[67894]: audit 2024-07-28T19:42:48.707527+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:42:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:49 smithi138 bash[67894]: audit 2024-07-28T19:42:48.708232+0000 mon.a (mon.0) 994 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-07-28T19:42:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:49 smithi138 bash[67894]: audit 2024-07-28T19:42:48.909198+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:49 smithi138 bash[67894]: audit 2024-07-28T19:42:48.913877+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:49.927 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -e sha1=1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64 -- bash -c 'ceph versions' 2024-07-28T19:42:50.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[82644]: audit 2024-07-28T19:42:49.191626+0000 mgr.y (mgr.25358) 306 : audit [DBG] from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[82644]: audit 2024-07-28T19:42:49.321829+0000 mon.a (mon.0) 997 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-28T19:42:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[82644]: cluster 2024-07-28T19:42:49.321896+0000 mon.a (mon.0) 998 : cluster [DBG] osdmap e651: 8 total, 7 up, 8 in 2024-07-28T19:42:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[82644]: audit 2024-07-28T19:42:49.327682+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:42:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[82644]: audit 2024-07-28T19:42:49.328511+0000 mon.a (mon.0) 999 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:42:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[80199]: debug 2024-07-28T19:42:50.314+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[80199]: audit 2024-07-28T19:42:49.191626+0000 mgr.y (mgr.25358) 306 : audit [DBG] from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[80199]: audit 2024-07-28T19:42:49.321829+0000 mon.a (mon.0) 997 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-28T19:42:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[80199]: cluster 2024-07-28T19:42:49.321896+0000 mon.a (mon.0) 998 : cluster [DBG] osdmap e651: 8 total, 7 up, 8 in 2024-07-28T19:42:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[80199]: audit 2024-07-28T19:42:49.327682+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:42:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:50 smithi100 bash[80199]: audit 2024-07-28T19:42:49.328511+0000 mon.a (mon.0) 999 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:42:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:50 smithi138 bash[67894]: audit 2024-07-28T19:42:49.191626+0000 mgr.y (mgr.25358) 306 : audit [DBG] from='client.34518 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-07-28T19:42:50.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:50 smithi138 bash[67894]: audit 2024-07-28T19:42:49.321829+0000 mon.a (mon.0) 997 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-07-28T19:42:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:50 smithi138 bash[67894]: cluster 2024-07-28T19:42:49.321896+0000 mon.a (mon.0) 998 : cluster [DBG] osdmap e651: 8 total, 7 up, 8 in 2024-07-28T19:42:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:50 smithi138 bash[67894]: audit 2024-07-28T19:42:49.327682+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:42:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:50 smithi138 bash[67894]: audit 2024-07-28T19:42:49.328511+0000 mon.a (mon.0) 999 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2024-07-28T19:42:51.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[80199]: cluster 2024-07-28T19:42:49.768760+0000 mgr.y (mgr.25358) 307 : cluster [DBG] pgmap v567: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 811 KiB/s rd, 389 KiB/s wr, 787 op/s; 77/537 objects degraded (14.339%) 2024-07-28T19:42:51.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[80199]: cluster 2024-07-28T19:42:50.319482+0000 mon.a (mon.0) 1000 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:51.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[80199]: cluster 2024-07-28T19:42:50.319502+0000 mon.a (mon.0) 1001 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:42:51.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[80199]: cluster 2024-07-28T19:42:50.319898+0000 mon.a (mon.0) 1002 : cluster [WRN] Health check update: Degraded data redundancy: 77/537 objects degraded (14.339%), 53 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[80199]: cluster 2024-07-28T19:42:50.326158+0000 mon.a (mon.0) 1003 : cluster [INF] osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885] boot 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[80199]: cluster 2024-07-28T19:42:50.326208+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[80199]: audit 2024-07-28T19:42:50.327832+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[80199]: cluster 2024-07-28T19:42:51.328589+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[82644]: cluster 2024-07-28T19:42:49.768760+0000 mgr.y (mgr.25358) 307 : cluster [DBG] pgmap v567: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 811 KiB/s rd, 389 KiB/s wr, 787 op/s; 77/537 objects degraded (14.339%) 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[82644]: cluster 2024-07-28T19:42:50.319482+0000 mon.a (mon.0) 1000 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[82644]: cluster 2024-07-28T19:42:50.319502+0000 mon.a (mon.0) 1001 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[82644]: cluster 2024-07-28T19:42:50.319898+0000 mon.a (mon.0) 1002 : cluster [WRN] Health check update: Degraded data redundancy: 77/537 objects degraded (14.339%), 53 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[82644]: cluster 2024-07-28T19:42:50.326158+0000 mon.a (mon.0) 1003 : cluster [INF] osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885] boot 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[82644]: cluster 2024-07-28T19:42:50.326208+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-07-28T19:42:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[82644]: audit 2024-07-28T19:42:50.327832+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:42:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:51 smithi100 bash[82644]: cluster 2024-07-28T19:42:51.328589+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-07-28T19:42:51.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:51 smithi138 bash[67894]: cluster 2024-07-28T19:42:49.768760+0000 mgr.y (mgr.25358) 307 : cluster [DBG] pgmap v567: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 811 KiB/s rd, 389 KiB/s wr, 787 op/s; 77/537 objects degraded (14.339%) 2024-07-28T19:42:51.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:51 smithi138 bash[67894]: cluster 2024-07-28T19:42:50.319482+0000 mon.a (mon.0) 1000 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-07-28T19:42:51.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:51 smithi138 bash[67894]: cluster 2024-07-28T19:42:50.319502+0000 mon.a (mon.0) 1001 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-07-28T19:42:51.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:51 smithi138 bash[67894]: cluster 2024-07-28T19:42:50.319898+0000 mon.a (mon.0) 1002 : cluster [WRN] Health check update: Degraded data redundancy: 77/537 objects degraded (14.339%), 53 pgs degraded (PG_DEGRADED) 2024-07-28T19:42:51.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:51 smithi138 bash[67894]: cluster 2024-07-28T19:42:50.326158+0000 mon.a (mon.0) 1003 : cluster [INF] osd.4 [v2:172.21.15.138:6800/3051177885,v1:172.21.15.138:6801/3051177885] boot 2024-07-28T19:42:51.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:51 smithi138 bash[67894]: cluster 2024-07-28T19:42:50.326208+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-07-28T19:42:51.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:51 smithi138 bash[67894]: audit 2024-07-28T19:42:50.327832+0000 mon.a (mon.0) 1005 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-07-28T19:42:51.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:51 smithi138 bash[67894]: cluster 2024-07-28T19:42:51.328589+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-07-28T19:42:52.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:42:52 smithi138 bash[60615]: ts=2024-07-28T19:42:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:42:53.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:42:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:42:52] "GET /metrics HTTP/1.1" 200 60627 "" "Prometheus/2.43.0" 2024-07-28T19:42:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:53 smithi100 bash[80199]: cluster 2024-07-28T19:42:51.769890+0000 mgr.y (mgr.25358) 308 : cluster [DBG] pgmap v570: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 517 KiB/s rd, 212 KiB/s wr, 472 op/s; 77/537 objects degraded (14.339%) 2024-07-28T19:42:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:53 smithi100 bash[80199]: cluster 2024-07-28T19:42:52.358076+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-07-28T19:42:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:53 smithi100 bash[80199]: audit 2024-07-28T19:42:52.640430+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:53 smithi100 bash[80199]: audit 2024-07-28T19:42:52.642215+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:53 smithi100 bash[82644]: cluster 2024-07-28T19:42:51.769890+0000 mgr.y (mgr.25358) 308 : cluster [DBG] pgmap v570: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 517 KiB/s rd, 212 KiB/s wr, 472 op/s; 77/537 objects degraded (14.339%) 2024-07-28T19:42:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:53 smithi100 bash[82644]: cluster 2024-07-28T19:42:52.358076+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-07-28T19:42:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:53 smithi100 bash[82644]: audit 2024-07-28T19:42:52.640430+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:53 smithi100 bash[82644]: audit 2024-07-28T19:42:52.642215+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:53 smithi138 bash[67894]: cluster 2024-07-28T19:42:51.769890+0000 mgr.y (mgr.25358) 308 : cluster [DBG] pgmap v570: 425 pgs: 8 active+clean+remapped, 103 active+undersized, 53 active+undersized+degraded, 261 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 517 KiB/s rd, 212 KiB/s wr, 472 op/s; 77/537 objects degraded (14.339%) 2024-07-28T19:42:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:53 smithi138 bash[67894]: cluster 2024-07-28T19:42:52.358076+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-07-28T19:42:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:53 smithi138 bash[67894]: audit 2024-07-28T19:42:52.640430+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:42:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:53 smithi138 bash[67894]: audit 2024-07-28T19:42:52.642215+0000 mon.a (mon.0) 1009 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:42:54.704 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:42:55.717 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:55 smithi138 bash[67894]: cluster 2024-07-28T19:42:53.771500+0000 mgr.y (mgr.25358) 309 : cluster [DBG] pgmap v572: 425 pgs: 5 active+clean+remapped, 64 active+undersized, 21 active+undersized+degraded, 335 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 920 B/s wr, 10 op/s; 36/543 objects degraded (6.630%); 47 KiB/s, 0 objects/s recovering 2024-07-28T19:42:55.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:55 smithi100 bash[80199]: cluster 2024-07-28T19:42:53.771500+0000 mgr.y (mgr.25358) 309 : cluster [DBG] pgmap v572: 425 pgs: 5 active+clean+remapped, 64 active+undersized, 21 active+undersized+degraded, 335 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 920 B/s wr, 10 op/s; 36/543 objects degraded (6.630%); 47 KiB/s, 0 objects/s recovering 2024-07-28T19:42:55.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:55 smithi100 bash[82644]: cluster 2024-07-28T19:42:53.771500+0000 mgr.y (mgr.25358) 309 : cluster [DBG] pgmap v572: 425 pgs: 5 active+clean+remapped, 64 active+undersized, 21 active+undersized+degraded, 335 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 920 B/s wr, 10 op/s; 36/543 objects degraded (6.630%); 47 KiB/s, 0 objects/s recovering 2024-07-28T19:42:56.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:56 smithi100 bash[82644]: cluster 2024-07-28T19:42:55.772805+0000 mgr.y (mgr.25358) 310 : cluster [DBG] pgmap v573: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 682 B/s wr, 7 op/s; 35 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:56.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:56 smithi100 bash[82644]: cluster 2024-07-28T19:42:56.358145+0000 mon.a (mon.0) 1010 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/543 objects degraded (6.630%), 21 pgs degraded) 2024-07-28T19:42:56.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:56 smithi100 bash[80199]: cluster 2024-07-28T19:42:55.772805+0000 mgr.y (mgr.25358) 310 : cluster [DBG] pgmap v573: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 682 B/s wr, 7 op/s; 35 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:56.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:56 smithi100 bash[80199]: cluster 2024-07-28T19:42:56.358145+0000 mon.a (mon.0) 1010 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/543 objects degraded (6.630%), 21 pgs degraded) 2024-07-28T19:42:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:56 smithi138 bash[67894]: cluster 2024-07-28T19:42:55.772805+0000 mgr.y (mgr.25358) 310 : cluster [DBG] pgmap v573: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 682 B/s wr, 7 op/s; 35 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:56.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:56 smithi138 bash[67894]: cluster 2024-07-28T19:42:56.358145+0000 mon.a (mon.0) 1010 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/543 objects degraded (6.630%), 21 pgs degraded) 2024-07-28T19:42:58.607 INFO:teuthology.orchestra.run.smithi100.stdout:{ 2024-07-28T19:42:58.607 INFO:teuthology.orchestra.run.smithi100.stdout: "mon": { 2024-07-28T19:42:58.607 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable)": 3 2024-07-28T19:42:58.607 INFO:teuthology.orchestra.run.smithi100.stdout: }, 2024-07-28T19:42:58.607 INFO:teuthology.orchestra.run.smithi100.stdout: "mgr": { 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable)": 2 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: }, 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "osd": { 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)": 3, 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable)": 5 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: }, 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "mds": { 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)": 2 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: }, 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "overall": { 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 17.2.7-1155-g3120d060 (3120d0609b388fe36c245c2679b9cdf0554c9d32) quincy (stable)": 5, 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph version 18.2.2-1765-g1c7dd747 (1c7dd7472a8aa5756d215ece8ec9c4ea9b1c4e64) reef (stable)": 10 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout: } 2024-07-28T19:42:58.608 INFO:teuthology.orchestra.run.smithi100.stdout:} 2024-07-28T19:42:58.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:58 smithi100 bash[82644]: cluster 2024-07-28T19:42:57.774340+0000 mgr.y (mgr.25358) 311 : cluster [DBG] pgmap v574: 425 pgs: 425 active+clean; 15 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 687 B/s rd, 550 KiB/s wr, 1 op/s; 28 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:58.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:42:58 smithi100 bash[82644]: audit 2024-07-28T19:42:58.609720+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.100:0/1801133488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:58.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:58 smithi100 bash[80199]: cluster 2024-07-28T19:42:57.774340+0000 mgr.y (mgr.25358) 311 : cluster [DBG] pgmap v574: 425 pgs: 425 active+clean; 15 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 687 B/s rd, 550 KiB/s wr, 1 op/s; 28 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:58.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:42:58 smithi100 bash[80199]: audit 2024-07-28T19:42:58.609720+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.100:0/1801133488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:59.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:58 smithi138 bash[67894]: cluster 2024-07-28T19:42:57.774340+0000 mgr.y (mgr.25358) 311 : cluster [DBG] pgmap v574: 425 pgs: 425 active+clean; 15 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 687 B/s rd, 550 KiB/s wr, 1 op/s; 28 KiB/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:42:59.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:42:58 smithi138 bash[67894]: audit 2024-07-28T19:42:58.609720+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.100:0/1801133488' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-07-28T19:42:59.438 DEBUG:teuthology.parallel:result is None 2024-07-28T19:43:00.603 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:00.603 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:00.603 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:00.604 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.605 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.606 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.607 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.608 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:00.609 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 512~512 = 512 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:aio discard: 1536~512 = 0 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.610 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:43:00.611 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:43:00.612 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.613 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.614 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.615 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.616 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.617 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.618 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:00.619 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:00.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.420578+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.431416+0000 mon.a (mon.0) 1013 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.433780+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.435074+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.443869+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.559291+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.561736+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.562901+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: audit 2024-07-28T19:42:59.571641+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[80199]: cluster 2024-07-28T19:42:59.777093+0000 mgr.y (mgr.25358) 312 : cluster [DBG] pgmap v575: 425 pgs: 425 active+clean; 23 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 430 op/s; 25 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.420578+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.431416+0000 mon.a (mon.0) 1013 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.433780+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:43:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.435074+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:43:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.443869+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.559291+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:43:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.561736+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:43:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.562901+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:43:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: audit 2024-07-28T19:42:59.571641+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:00 smithi100 bash[82644]: cluster 2024-07-28T19:42:59.777093+0000 mgr.y (mgr.25358) 312 : cluster [DBG] pgmap v575: 425 pgs: 425 active+clean; 23 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 430 op/s; 25 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:00.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.420578+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.431416+0000 mon.a (mon.0) 1013 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.433780+0000 mon.a (mon.0) 1014 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:43:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.435074+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:43:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.443869+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.559291+0000 mon.a (mon.0) 1017 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:43:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.561736+0000 mon.a (mon.0) 1018 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:43:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.562901+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:43:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: audit 2024-07-28T19:42:59.571641+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:00 smithi138 bash[67894]: cluster 2024-07-28T19:42:59.777093+0000 mgr.y (mgr.25358) 312 : cluster [DBG] pgmap v575: 425 pgs: 425 active+clean; 23 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.7 MiB/s wr, 430 op/s; 25 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:43:02 smithi138 bash[60615]: ts=2024-07-28T19:43:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:43:03.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:02 smithi100 bash[80199]: cluster 2024-07-28T19:43:01.778472+0000 mgr.y (mgr.25358) 313 : cluster [DBG] pgmap v576: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.8 MiB/s wr, 363 op/s; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:03.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:43:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:43:02] "GET /metrics HTTP/1.1" 200 60627 "" "Prometheus/2.43.0" 2024-07-28T19:43:03.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:02 smithi100 bash[82644]: cluster 2024-07-28T19:43:01.778472+0000 mgr.y (mgr.25358) 313 : cluster [DBG] pgmap v576: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.8 MiB/s wr, 363 op/s; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:03.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:02 smithi138 bash[67894]: cluster 2024-07-28T19:43:01.778472+0000 mgr.y (mgr.25358) 313 : cluster [DBG] pgmap v576: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.8 MiB/s wr, 363 op/s; 21 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:04.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:03 smithi100 bash[82644]: audit 2024-07-28T19:43:02.972684+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:04.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:03 smithi100 bash[80199]: audit 2024-07-28T19:43:02.972684+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:04.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:03 smithi138 bash[67894]: audit 2024-07-28T19:43:02.972684+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:05.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:04 smithi100 bash[80199]: cluster 2024-07-28T19:43:03.781184+0000 mgr.y (mgr.25358) 314 : cluster [DBG] pgmap v577: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 318 op/s; 18 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:05.250 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:04 smithi100 bash[82644]: cluster 2024-07-28T19:43:03.781184+0000 mgr.y (mgr.25358) 314 : cluster [DBG] pgmap v577: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 318 op/s; 18 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:05.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:04 smithi138 bash[67894]: cluster 2024-07-28T19:43:03.781184+0000 mgr.y (mgr.25358) 314 : cluster [DBG] pgmap v577: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 318 op/s; 18 KiB/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:07.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:06 smithi100 bash[80199]: cluster 2024-07-28T19:43:05.783381+0000 mgr.y (mgr.25358) 315 : cluster [DBG] pgmap v578: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 599 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:07.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:06 smithi100 bash[82644]: cluster 2024-07-28T19:43:05.783381+0000 mgr.y (mgr.25358) 315 : cluster [DBG] pgmap v578: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 599 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:07.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:06 smithi138 bash[67894]: cluster 2024-07-28T19:43:05.783381+0000 mgr.y (mgr.25358) 315 : cluster [DBG] pgmap v578: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 599 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:08.274 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.274 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.274 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.274 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.274 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.275 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.276 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.276 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:08.276 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.276 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.276 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:08.276 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.277 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.277 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.277 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.277 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.277 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.277 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.277 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.278 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.278 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.278 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.278 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.278 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.278 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.279 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.280 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.280 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.280 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.280 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.280 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.281 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.282 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.283 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.284 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.285 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:08.286 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 4 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:read: 4 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (19330 ms) 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-07-28T19:43:08.287 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:08.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:08 smithi100 bash[80199]: audit 2024-07-28T19:43:07.639915+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:08.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:08 smithi100 bash[80199]: audit 2024-07-28T19:43:07.641526+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:08.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:08 smithi100 bash[80199]: cluster 2024-07-28T19:43:07.784382+0000 mgr.y (mgr.25358) 316 : cluster [DBG] pgmap v579: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 599 op/s 2024-07-28T19:43:08.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:08 smithi100 bash[82644]: audit 2024-07-28T19:43:07.639915+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:08.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:08 smithi100 bash[82644]: audit 2024-07-28T19:43:07.641526+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:08.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:08 smithi100 bash[82644]: cluster 2024-07-28T19:43:07.784382+0000 mgr.y (mgr.25358) 316 : cluster [DBG] pgmap v579: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 599 op/s 2024-07-28T19:43:09.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:08 smithi138 bash[67894]: audit 2024-07-28T19:43:07.639915+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:09.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:08 smithi138 bash[67894]: audit 2024-07-28T19:43:07.641526+0000 mon.a (mon.0) 1023 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:09.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:08 smithi138 bash[67894]: cluster 2024-07-28T19:43:07.784382+0000 mgr.y (mgr.25358) 316 : cluster [DBG] pgmap v579: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.6 MiB/s wr, 599 op/s 2024-07-28T19:43:11.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:10 smithi100 bash[80199]: cluster 2024-07-28T19:43:09.787437+0000 mgr.y (mgr.25358) 317 : cluster [DBG] pgmap v580: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.4 MiB/s wr, 783 op/s 2024-07-28T19:43:11.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:10 smithi100 bash[82644]: cluster 2024-07-28T19:43:09.787437+0000 mgr.y (mgr.25358) 317 : cluster [DBG] pgmap v580: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.4 MiB/s wr, 783 op/s 2024-07-28T19:43:11.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:10 smithi138 bash[67894]: cluster 2024-07-28T19:43:09.787437+0000 mgr.y (mgr.25358) 317 : cluster [DBG] pgmap v580: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.4 MiB/s wr, 783 op/s 2024-07-28T19:43:12.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:43:12 smithi138 bash[60615]: ts=2024-07-28T19:43:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:43:13.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:12 smithi100 bash[80199]: cluster 2024-07-28T19:43:11.788802+0000 mgr.y (mgr.25358) 318 : cluster [DBG] pgmap v581: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 560 KiB/s wr, 482 op/s 2024-07-28T19:43:13.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:12 smithi100 bash[82644]: cluster 2024-07-28T19:43:11.788802+0000 mgr.y (mgr.25358) 318 : cluster [DBG] pgmap v581: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 560 KiB/s wr, 482 op/s 2024-07-28T19:43:13.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:43:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:43:12] "GET /metrics HTTP/1.1" 200 60764 "" "Prometheus/2.43.0" 2024-07-28T19:43:13.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:12 smithi138 bash[67894]: cluster 2024-07-28T19:43:11.788802+0000 mgr.y (mgr.25358) 318 : cluster [DBG] pgmap v581: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 560 KiB/s wr, 482 op/s 2024-07-28T19:43:13.436 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:13.436 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:13.436 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:13.436 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:13.436 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.437 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:13.438 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.439 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:13.440 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:13.441 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.442 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 512~512 = 512 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:aio discard: 1536~512 = 0 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.443 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.444 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.445 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.446 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:43:13.447 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.448 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.449 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:13.450 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:13.451 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:15.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:14 smithi100 bash[80199]: cluster 2024-07-28T19:43:13.791932+0000 mgr.y (mgr.25358) 319 : cluster [DBG] pgmap v582: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 219 KiB/s wr, 482 op/s 2024-07-28T19:43:15.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:14 smithi100 bash[82644]: cluster 2024-07-28T19:43:13.791932+0000 mgr.y (mgr.25358) 319 : cluster [DBG] pgmap v582: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 219 KiB/s wr, 482 op/s 2024-07-28T19:43:15.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:14 smithi138 bash[67894]: cluster 2024-07-28T19:43:13.791932+0000 mgr.y (mgr.25358) 319 : cluster [DBG] pgmap v582: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 219 KiB/s wr, 482 op/s 2024-07-28T19:43:17.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:16 smithi100 bash[80199]: cluster 2024-07-28T19:43:15.794297+0000 mgr.y (mgr.25358) 320 : cluster [DBG] pgmap v583: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 305 KiB/s wr, 842 op/s 2024-07-28T19:43:17.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:16 smithi100 bash[82644]: cluster 2024-07-28T19:43:15.794297+0000 mgr.y (mgr.25358) 320 : cluster [DBG] pgmap v583: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 305 KiB/s wr, 842 op/s 2024-07-28T19:43:17.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:16 smithi138 bash[67894]: cluster 2024-07-28T19:43:15.794297+0000 mgr.y (mgr.25358) 320 : cluster [DBG] pgmap v583: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 305 KiB/s wr, 842 op/s 2024-07-28T19:43:17.857 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.857 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.857 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.857 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.857 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.857 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.857 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.858 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.859 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.860 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.861 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.862 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:17.863 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.864 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:17.865 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (9583 ms) 2024-07-28T19:43:17.866 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-07-28T19:43:19.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:18 smithi100 bash[80199]: cluster 2024-07-28T19:43:17.796063+0000 mgr.y (mgr.25358) 321 : cluster [DBG] pgmap v584: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 747 KiB/s rd, 160 KiB/s wr, 544 op/s 2024-07-28T19:43:19.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:18 smithi100 bash[82644]: cluster 2024-07-28T19:43:17.796063+0000 mgr.y (mgr.25358) 321 : cluster [DBG] pgmap v584: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 747 KiB/s rd, 160 KiB/s wr, 544 op/s 2024-07-28T19:43:19.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:18 smithi138 bash[67894]: cluster 2024-07-28T19:43:17.796063+0000 mgr.y (mgr.25358) 321 : cluster [DBG] pgmap v584: 425 pgs: 425 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 747 KiB/s rd, 160 KiB/s wr, 544 op/s 2024-07-28T19:43:19.884 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:20.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:19 smithi100 bash[80199]: cluster 2024-07-28T19:43:18.877794+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-07-28T19:43:20.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:19 smithi100 bash[80199]: audit 2024-07-28T19:43:18.885654+0000 mon.a (mon.0) 1025 : audit [INF] from='client.? 172.21.15.100:0/2202971735' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:20.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:19 smithi100 bash[80199]: audit 2024-07-28T19:43:19.876834+0000 mon.a (mon.0) 1026 : audit [INF] from='client.? 172.21.15.100:0/2202971735' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:20.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:19 smithi100 bash[80199]: cluster 2024-07-28T19:43:19.877412+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-07-28T19:43:20.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:19 smithi100 bash[82644]: cluster 2024-07-28T19:43:18.877794+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-07-28T19:43:20.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:19 smithi100 bash[82644]: audit 2024-07-28T19:43:18.885654+0000 mon.a (mon.0) 1025 : audit [INF] from='client.? 172.21.15.100:0/2202971735' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:20.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:19 smithi100 bash[82644]: audit 2024-07-28T19:43:19.876834+0000 mon.a (mon.0) 1026 : audit [INF] from='client.? 172.21.15.100:0/2202971735' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:20.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:19 smithi100 bash[82644]: cluster 2024-07-28T19:43:19.877412+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-07-28T19:43:20.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:19 smithi138 bash[67894]: cluster 2024-07-28T19:43:18.877794+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-07-28T19:43:20.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:19 smithi138 bash[67894]: audit 2024-07-28T19:43:18.885654+0000 mon.a (mon.0) 1025 : audit [INF] from='client.? 172.21.15.100:0/2202971735' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:20.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:19 smithi138 bash[67894]: audit 2024-07-28T19:43:19.876834+0000 mon.a (mon.0) 1026 : audit [INF] from='client.? 172.21.15.100:0/2202971735' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:20.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:19 smithi138 bash[67894]: cluster 2024-07-28T19:43:19.877412+0000 mon.a (mon.0) 1027 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-07-28T19:43:21.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:20 smithi100 bash[80199]: cluster 2024-07-28T19:43:19.798022+0000 mgr.y (mgr.25358) 322 : cluster [DBG] pgmap v586: 457 pgs: 8 creating+peering, 24 unknown, 425 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 405 KiB/s rd, 186 KiB/s wr, 769 op/s 2024-07-28T19:43:21.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:20 smithi100 bash[80199]: cluster 2024-07-28T19:43:19.886265+0000 mon.a (mon.0) 1028 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:43:21.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:20 smithi100 bash[80199]: cluster 2024-07-28T19:43:20.902214+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-07-28T19:43:21.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:20 smithi100 bash[82644]: cluster 2024-07-28T19:43:19.798022+0000 mgr.y (mgr.25358) 322 : cluster [DBG] pgmap v586: 457 pgs: 8 creating+peering, 24 unknown, 425 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 405 KiB/s rd, 186 KiB/s wr, 769 op/s 2024-07-28T19:43:21.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:20 smithi100 bash[82644]: cluster 2024-07-28T19:43:19.886265+0000 mon.a (mon.0) 1028 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:43:21.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:20 smithi100 bash[82644]: cluster 2024-07-28T19:43:20.902214+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-07-28T19:43:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:20 smithi138 bash[67894]: cluster 2024-07-28T19:43:19.798022+0000 mgr.y (mgr.25358) 322 : cluster [DBG] pgmap v586: 457 pgs: 8 creating+peering, 24 unknown, 425 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 405 KiB/s rd, 186 KiB/s wr, 769 op/s 2024-07-28T19:43:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:20 smithi138 bash[67894]: cluster 2024-07-28T19:43:19.886265+0000 mon.a (mon.0) 1028 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:43:21.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:20 smithi138 bash[67894]: cluster 2024-07-28T19:43:20.902214+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:22.060 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:22.061 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.062 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:22.063 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 512~512 = 512 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:aio discard: 1536~512 = 0 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:22.064 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:22.065 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestDataPoolIO (4203 ms) 2024-07-28T19:43:22.065 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-07-28T19:43:22.065 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:22.131 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (71 ms) 2024-07-28T19:43:22.131 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-07-28T19:43:22.131 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:22.200 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (69 ms) 2024-07-28T19:43:22.200 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-07-28T19:43:22.200 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:22.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:21 smithi100 bash[80199]: cluster 2024-07-28T19:43:21.914274+0000 mon.a (mon.0) 1030 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:43:22.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:21 smithi100 bash[80199]: cluster 2024-07-28T19:43:21.929690+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-07-28T19:43:22.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:21 smithi100 bash[82644]: cluster 2024-07-28T19:43:21.914274+0000 mon.a (mon.0) 1030 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:43:22.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:21 smithi100 bash[82644]: cluster 2024-07-28T19:43:21.929690+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-07-28T19:43:22.267 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (67 ms) 2024-07-28T19:43:22.267 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-07-28T19:43:22.267 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:22.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:21 smithi138 bash[67894]: cluster 2024-07-28T19:43:21.914274+0000 mon.a (mon.0) 1030 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:43:22.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:21 smithi138 bash[67894]: cluster 2024-07-28T19:43:21.929690+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-07-28T19:43:22.333 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (67 ms) 2024-07-28T19:43:22.334 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-07-28T19:43:22.334 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:22.483 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (149 ms) 2024-07-28T19:43:22.483 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-07-28T19:43:22.484 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:22.621 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (138 ms) 2024-07-28T19:43:22.621 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-07-28T19:43:22.621 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:22.751 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (130 ms) 2024-07-28T19:43:22.751 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-07-28T19:43:22.751 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:22.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:43:22 smithi138 bash[60615]: ts=2024-07-28T19:43:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:43:22.872 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (122 ms) 2024-07-28T19:43:22.872 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-07-28T19:43:22.872 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:23.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:22 smithi100 bash[82644]: cluster 2024-07-28T19:43:21.799034+0000 mgr.y (mgr.25358) 323 : cluster [DBG] pgmap v589: 457 pgs: 8 creating+peering, 23 unknown, 426 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 272 KiB/s rd, 136 KiB/s wr, 562 op/s 2024-07-28T19:43:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:22 smithi100 bash[82644]: audit 2024-07-28T19:43:22.635931+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:23.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:43:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:43:22] "GET /metrics HTTP/1.1" 200 63130 "" "Prometheus/2.43.0" 2024-07-28T19:43:23.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:22 smithi100 bash[80199]: cluster 2024-07-28T19:43:21.799034+0000 mgr.y (mgr.25358) 323 : cluster [DBG] pgmap v589: 457 pgs: 8 creating+peering, 23 unknown, 426 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 272 KiB/s rd, 136 KiB/s wr, 562 op/s 2024-07-28T19:43:23.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:22 smithi100 bash[80199]: audit 2024-07-28T19:43:22.635931+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:23.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:22 smithi138 bash[67894]: cluster 2024-07-28T19:43:21.799034+0000 mgr.y (mgr.25358) 323 : cluster [DBG] pgmap v589: 457 pgs: 8 creating+peering, 23 unknown, 426 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 272 KiB/s rd, 136 KiB/s wr, 562 op/s 2024-07-28T19:43:23.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:22 smithi138 bash[67894]: audit 2024-07-28T19:43:22.635931+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:23.327 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (455 ms) 2024-07-28T19:43:23.328 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-07-28T19:43:23.328 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:23.838 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (510 ms) 2024-07-28T19:43:23.838 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-07-28T19:43:23.838 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:23.901 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (64 ms) 2024-07-28T19:43:23.902 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-07-28T19:43:23.902 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:23.971 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (70 ms) 2024-07-28T19:43:23.972 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-07-28T19:43:23.972 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:24.039 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (67 ms) 2024-07-28T19:43:24.039 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-07-28T19:43:24.039 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:24.101 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (63 ms) 2024-07-28T19:43:24.101 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-07-28T19:43:24.102 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:24.159 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.159 INFO:tasks.workunit.client.0.smithi100.stdout:aio discard: 0~1048576 = 0 2024-07-28T19:43:24.160 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.160 INFO:tasks.workunit.client.0.smithi100.stdout:aio discard: 0~4194304 = 0 2024-07-28T19:43:24.160 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.160 INFO:tasks.workunit.client.0.smithi100.stdout:aio discard: 3145728~1048576 = 0 2024-07-28T19:43:24.160 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (57 ms) 2024-07-28T19:43:24.160 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestFUA 2024-07-28T19:43:24.160 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:24.447 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.448 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.449 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.450 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.451 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.452 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.452 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.452 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.452 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.452 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.452 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestFUA (289 ms) 2024-07-28T19:43:24.452 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-07-28T19:43:24.452 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.497 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.498 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.499 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.499 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.499 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.499 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:24.499 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:24.531 INFO:tasks.workunit.client.0.smithi100.stdout:start compare and write 2024-07-28T19:43:24.532 INFO:tasks.workunit.client.0.smithi100.stdout:compare and wrote: 512 2024-07-28T19:43:24.532 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.532 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:24.532 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.533 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.533 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:24.533 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.533 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:24.533 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.536 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.536 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:24.536 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.536 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:24.536 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.540 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.540 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:24.540 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.540 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:24.540 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.543 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.543 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:24.543 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.543 INFO:tasks.workunit.client.0.smithi100.stdout:started aio compare and write 2024-07-28T19:43:24.543 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:24.548 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:24.548 INFO:tasks.workunit.client.0.smithi100.stdout:finished aio compare and write 2024-07-28T19:43:24.548 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.548 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.548 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.548 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.549 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.550 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.550 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:24.550 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.550 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.550 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.551 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.551 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:24.551 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.551 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.551 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.552 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.552 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:24.552 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:24.552 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:24.552 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:24.552 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:24.552 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:24.552 INFO:tasks.workunit.client.0.smithi100.stdout:discard: 512~512 2024-07-28T19:43:24.553 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:25.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:24 smithi100 bash[80199]: cluster 2024-07-28T19:43:23.801913+0000 mgr.y (mgr.25358) 324 : cluster [DBG] pgmap v591: 457 pgs: 8 creating+peering, 449 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.9 MiB/s wr, 772 op/s 2024-07-28T19:43:25.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:24 smithi100 bash[82644]: cluster 2024-07-28T19:43:23.801913+0000 mgr.y (mgr.25358) 324 : cluster [DBG] pgmap v591: 457 pgs: 8 creating+peering, 449 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.9 MiB/s wr, 772 op/s 2024-07-28T19:43:25.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:24 smithi138 bash[67894]: cluster 2024-07-28T19:43:23.801913+0000 mgr.y (mgr.25358) 324 : cluster [DBG] pgmap v591: 457 pgs: 8 creating+peering, 449 active+clean; 63 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.9 MiB/s wr, 772 op/s 2024-07-28T19:43:27.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:26 smithi100 bash[82644]: cluster 2024-07-28T19:43:25.804357+0000 mgr.y (mgr.25358) 325 : cluster [DBG] pgmap v592: 457 pgs: 457 active+clean; 83 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 8.7 MiB/s wr, 135 op/s 2024-07-28T19:43:27.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:26 smithi100 bash[80199]: cluster 2024-07-28T19:43:25.804357+0000 mgr.y (mgr.25358) 325 : cluster [DBG] pgmap v592: 457 pgs: 457 active+clean; 83 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 8.7 MiB/s wr, 135 op/s 2024-07-28T19:43:27.298 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:27.298 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:27.298 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:27.298 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:43:27.299 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.300 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.300 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.300 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.300 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:43:27.300 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.300 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.301 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:43:27.301 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.301 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.301 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:43:27.301 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.302 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.302 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:43:27.302 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.302 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.302 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.303 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.303 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:43:27.303 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.303 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.303 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:43:27.303 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.304 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.304 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:43:27.304 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.304 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.304 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:43:27.305 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.305 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.305 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.305 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.305 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:43:27.306 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.306 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.306 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:43:27.306 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.306 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.307 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:43:27.307 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.307 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.307 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:43:27.307 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.307 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.308 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.308 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.308 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:43:27.308 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.308 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.309 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:43:27.309 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.309 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.309 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:43:27.309 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.309 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.310 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:43:27.310 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.310 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.310 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.311 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.311 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.311 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.311 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.311 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.312 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.312 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.312 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.312 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.312 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.312 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.313 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.313 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.313 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.313 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.313 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.313 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.314 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.314 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:26 smithi138 bash[67894]: cluster 2024-07-28T19:43:25.804357+0000 mgr.y (mgr.25358) 325 : cluster [DBG] pgmap v592: 457 pgs: 457 active+clean; 83 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 8.7 MiB/s wr, 135 op/s 2024-07-28T19:43:27.325 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.325 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.326 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.326 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.326 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.326 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.326 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.348 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.348 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.348 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.348 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.349 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.421 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.421 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.421 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.421 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.421 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.422 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.422 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.476 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.476 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.477 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.477 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.477 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.477 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.477 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.564 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.564 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.564 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.564 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.564 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.564 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.564 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:27.644 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.645 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.645 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.768 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.768 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.768 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.768 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.768 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.768 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.768 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:27.891 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:27.891 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:27.891 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:27.892 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:27.892 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:27.892 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:27.892 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:28.039 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:28.039 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:28.039 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:28.039 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:28.039 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:28.040 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:28.040 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:28.173 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:28.174 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:28.371 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:28.371 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:28.371 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:28.371 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:28.371 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:28.371 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:28.371 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:28.555 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:28.555 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:28.555 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:28.555 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:28.556 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:28.556 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:28.556 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:28.802 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:28.802 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:28.802 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:28.803 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:28.803 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:28.803 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:28.803 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:29.002 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:29.002 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:29.002 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:29.003 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:29.235 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:29.235 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:29.235 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:29.235 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:29.235 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:29.235 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:29.235 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:29.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:28 smithi100 bash[80199]: cluster 2024-07-28T19:43:27.806098+0000 mgr.y (mgr.25358) 326 : cluster [DBG] pgmap v593: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.7 MiB/s wr, 638 op/s 2024-07-28T19:43:29.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:28 smithi100 bash[82644]: cluster 2024-07-28T19:43:27.806098+0000 mgr.y (mgr.25358) 326 : cluster [DBG] pgmap v593: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.7 MiB/s wr, 638 op/s 2024-07-28T19:43:29.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:28 smithi138 bash[67894]: cluster 2024-07-28T19:43:27.806098+0000 mgr.y (mgr.25358) 326 : cluster [DBG] pgmap v593: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.7 MiB/s wr, 638 op/s 2024-07-28T19:43:29.477 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:29.477 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:29.477 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:29.477 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:29.477 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:29.477 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:29.477 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:29.777 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:29.777 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:29.777 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:29.777 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:29.777 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:29.777 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:29.777 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:30.041 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:30.041 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:30.042 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:30.042 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:30.042 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:30.042 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:30.042 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:30.043 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:30.043 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:30.043 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:30.043 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestIOPP (5593 ms) 2024-07-28T19:43:30.043 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-07-28T19:43:30.043 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.107 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (67 ms) 2024-07-28T19:43:30.108 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-07-28T19:43:30.108 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.198 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (90 ms) 2024-07-28T19:43:30.199 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-07-28T19:43:30.199 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.254 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (56 ms) 2024-07-28T19:43:30.254 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-07-28T19:43:30.254 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.307 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (53 ms) 2024-07-28T19:43:30.307 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-07-28T19:43:30.307 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.367 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (60 ms) 2024-07-28T19:43:30.367 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-07-28T19:43:30.367 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.419 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:30.428 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (62 ms) 2024-07-28T19:43:30.429 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-07-28T19:43:30.429 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.495 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (66 ms) 2024-07-28T19:43:30.495 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-07-28T19:43:30.495 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.550 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:30.559 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (65 ms) 2024-07-28T19:43:30.559 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-07-28T19:43:30.559 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.625 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (65 ms) 2024-07-28T19:43:30.625 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-07-28T19:43:30.625 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.679 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:30.688 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (64 ms) 2024-07-28T19:43:30.689 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-07-28T19:43:30.689 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.760 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (71 ms) 2024-07-28T19:43:30.760 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-07-28T19:43:30.760 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.819 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:30.830 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (71 ms) 2024-07-28T19:43:30.831 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-07-28T19:43:30.831 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:30.981 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (150 ms) 2024-07-28T19:43:30.981 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-07-28T19:43:30.982 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:31.082 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:31.137 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (156 ms) 2024-07-28T19:43:31.137 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-07-28T19:43:31.137 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:31.301 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (164 ms) 2024-07-28T19:43:31.301 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-07-28T19:43:31.301 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:31.390 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:31.451 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (150 ms) 2024-07-28T19:43:31.451 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-07-28T19:43:31.451 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:31.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:31 smithi100 bash[80199]: cluster 2024-07-28T19:43:29.810030+0000 mgr.y (mgr.25358) 327 : cluster [DBG] pgmap v594: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.4 MiB/s wr, 616 op/s 2024-07-28T19:43:31.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:31 smithi100 bash[80199]: cluster 2024-07-28T19:43:30.135302+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-07-28T19:43:31.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:31 smithi100 bash[82644]: cluster 2024-07-28T19:43:29.810030+0000 mgr.y (mgr.25358) 327 : cluster [DBG] pgmap v594: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.4 MiB/s wr, 616 op/s 2024-07-28T19:43:31.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:31 smithi100 bash[82644]: cluster 2024-07-28T19:43:30.135302+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-07-28T19:43:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:31 smithi138 bash[67894]: cluster 2024-07-28T19:43:29.810030+0000 mgr.y (mgr.25358) 327 : cluster [DBG] pgmap v594: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.4 MiB/s wr, 616 op/s 2024-07-28T19:43:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:31 smithi138 bash[67894]: cluster 2024-07-28T19:43:30.135302+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-07-28T19:43:32.041 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (590 ms) 2024-07-28T19:43:32.041 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-07-28T19:43:32.041 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:32.524 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:32.568 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (527 ms) 2024-07-28T19:43:32.568 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-07-28T19:43:32.568 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:32.640 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:32.640 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:32.640 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:32.640 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:32.641 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:43:32.641 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.641 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:32.641 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:32.642 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:32.643 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:32.643 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.643 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:32.643 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:32.645 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:32.645 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:32.645 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.645 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:32.645 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:32.647 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:32.648 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:32.648 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.648 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:32.648 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:32.652 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:32.653 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:32.653 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.653 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:43:32.653 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:32.654 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:32.655 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:43:32.655 INFO:tasks.workunit.client.0.smithi100.stdout:read: 512 2024-07-28T19:43:32.655 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.655 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:32.655 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:32.656 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:32.656 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:32.656 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.656 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:32.656 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:32.657 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:32.657 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:32.657 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.657 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:32.657 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:32.658 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:32.658 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:32.658 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.658 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:32.658 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:32.659 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:32.659 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:32.659 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:32.659 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:43:32.659 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:43:32.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:43:32 smithi138 bash[60615]: ts=2024-07-28T19:43:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:43:33.133 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:43:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:43:32] "GET /metrics HTTP/1.1" 200 63130 "" "Prometheus/2.43.0" 2024-07-28T19:43:33.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:33 smithi100 bash[80199]: cluster 2024-07-28T19:43:31.811016+0000 mgr.y (mgr.25358) 328 : cluster [DBG] pgmap v596: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-07-28T19:43:33.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:33 smithi100 bash[80199]: audit 2024-07-28T19:43:33.062619+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:33.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:33 smithi100 bash[82644]: cluster 2024-07-28T19:43:31.811016+0000 mgr.y (mgr.25358) 328 : cluster [DBG] pgmap v596: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-07-28T19:43:33.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:33 smithi100 bash[82644]: audit 2024-07-28T19:43:33.062619+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:33 smithi138 bash[67894]: cluster 2024-07-28T19:43:31.811016+0000 mgr.y (mgr.25358) 328 : cluster [DBG] pgmap v596: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-07-28T19:43:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:33 smithi138 bash[67894]: audit 2024-07-28T19:43:33.062619+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:35.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:35 smithi100 bash[80199]: cluster 2024-07-28T19:43:33.813801+0000 mgr.y (mgr.25358) 329 : cluster [DBG] pgmap v597: 457 pgs: 457 active+clean; 115 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.0 MiB/s wr, 1.13k op/s 2024-07-28T19:43:35.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:35 smithi100 bash[82644]: cluster 2024-07-28T19:43:33.813801+0000 mgr.y (mgr.25358) 329 : cluster [DBG] pgmap v597: 457 pgs: 457 active+clean; 115 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.0 MiB/s wr, 1.13k op/s 2024-07-28T19:43:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:35 smithi138 bash[67894]: cluster 2024-07-28T19:43:33.813801+0000 mgr.y (mgr.25358) 329 : cluster [DBG] pgmap v597: 457 pgs: 457 active+clean; 115 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.0 MiB/s wr, 1.13k op/s 2024-07-28T19:43:36.493 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:43:36.493 INFO:tasks.workunit.client.0.smithi100.stdout:finished read 2024-07-28T19:43:36.493 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:43:36.493 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.493 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.493 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 0 2024-07-28T19:43:36.493 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 16384 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.494 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 49152 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 65536 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.495 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 98304 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 114688 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.496 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 147456 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 163840 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.497 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 196608 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.498 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 212992 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.499 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.500 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:36.500 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:36.500 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:36.500 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.500 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.500 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.500 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.500 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:36.524 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:36.524 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:36.524 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.524 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.524 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.525 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.525 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:36.549 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:36.549 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.550 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.551 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:36.633 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:36.633 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:36.633 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.633 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.633 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.633 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.633 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:36.706 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:36.706 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:36.706 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.706 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.706 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.706 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.707 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:36.819 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:36.820 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:36.820 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.820 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.820 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.820 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.820 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:36.920 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:36.920 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:36.920 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:36.920 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:36.921 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:37.103 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:37.103 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:37.103 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:37.103 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:37.103 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:37.103 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:37.103 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:37.269 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:37.269 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:37.269 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:37.270 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:37.270 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:37.270 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:37.270 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:37.491 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:37.491 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:37.491 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:37.491 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:37.491 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:37.492 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:37.492 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:37.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:37 smithi100 bash[80199]: cluster 2024-07-28T19:43:35.816115+0000 mgr.y (mgr.25358) 330 : cluster [DBG] pgmap v598: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.3 MiB/s wr, 1.14k op/s 2024-07-28T19:43:37.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:37 smithi100 bash[82644]: cluster 2024-07-28T19:43:35.816115+0000 mgr.y (mgr.25358) 330 : cluster [DBG] pgmap v598: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.3 MiB/s wr, 1.14k op/s 2024-07-28T19:43:37.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:37 smithi138 bash[67894]: cluster 2024-07-28T19:43:35.816115+0000 mgr.y (mgr.25358) 330 : cluster [DBG] pgmap v598: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.3 MiB/s wr, 1.14k op/s 2024-07-28T19:43:37.698 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:37.699 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:37.700 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:37.957 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:37.957 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:37.957 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:37.957 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:37.957 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:37.957 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:37.957 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:38.213 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:38.213 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:38.213 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:38.213 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:38.213 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:38.213 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:38.214 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:38.492 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:38.493 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:38.493 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:38.493 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:38.493 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:38.493 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:38.493 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:38.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.636451+0000 mon.a (mon.0) 1035 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.978551+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.979054+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.979537+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.16", "id": [7, 4]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.979960+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1f", "id": [7, 5]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.980403+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.980835+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.981287+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.981701+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1a", "id": [0, 2]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.982118+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]: dispatch 2024-07-28T19:43:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[80199]: audit 2024-07-28T19:43:37.982491+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.636451+0000 mon.a (mon.0) 1035 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.978551+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.979054+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.979537+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.16", "id": [7, 4]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.979960+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1f", "id": [7, 5]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.980403+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.980835+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.981287+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.981701+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1a", "id": [0, 2]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.982118+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]: dispatch 2024-07-28T19:43:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:38 smithi100 bash[82644]: audit 2024-07-28T19:43:37.982491+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]: dispatch 2024-07-28T19:43:38.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.636451+0000 mon.a (mon.0) 1035 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.978551+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.979054+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.979537+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.16", "id": [7, 4]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.979960+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1f", "id": [7, 5]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.980403+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.980835+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 0]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.981287+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.981701+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1a", "id": [0, 2]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.982118+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]: dispatch 2024-07-28T19:43:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:38 smithi138 bash[67894]: audit 2024-07-28T19:43:37.982491+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]: dispatch 2024-07-28T19:43:38.807 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:38.808 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:39.224 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:39.224 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:39.224 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:39.224 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:39.224 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:39.224 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:39.224 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:39.471 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:39.471 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:39.472 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:39.472 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:39.472 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:39.472 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:39.472 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:39.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: cluster 2024-07-28T19:43:37.817668+0000 mgr.y (mgr.25358) 331 : cluster [DBG] pgmap v599: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.1 MiB/s wr, 716 op/s 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.162897+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.163263+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 0]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.163701+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.16", "id": [7, 4]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.163988+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1f", "id": [7, 5]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.164459+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.164710+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 0]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.165240+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.165612+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1a", "id": [0, 2]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.165864+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]': finished 2024-07-28T19:43:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: audit 2024-07-28T19:43:38.166052+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]': finished 2024-07-28T19:43:39.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:39 smithi138 bash[67894]: cluster 2024-07-28T19:43:38.166174+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-07-28T19:43:39.576 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: cluster 2024-07-28T19:43:37.817668+0000 mgr.y (mgr.25358) 331 : cluster [DBG] pgmap v599: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.1 MiB/s wr, 716 op/s 2024-07-28T19:43:39.576 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.162897+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.163263+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 0]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.163701+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.16", "id": [7, 4]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.163988+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1f", "id": [7, 5]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.164459+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.164710+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 0]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.165240+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.165612+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1a", "id": [0, 2]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.165864+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: audit 2024-07-28T19:43:38.166052+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]': finished 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[80199]: cluster 2024-07-28T19:43:38.166174+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-07-28T19:43:39.577 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: cluster 2024-07-28T19:43:37.817668+0000 mgr.y (mgr.25358) 331 : cluster [DBG] pgmap v599: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.1 MiB/s wr, 716 op/s 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.162897+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.163263+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 0]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.163701+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.16", "id": [7, 4]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.163988+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1f", "id": [7, 5]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.164459+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.8", "id": [7, 0]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.164710+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 0]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.165240+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.165612+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1a", "id": [0, 2]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.165864+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: audit 2024-07-28T19:43:38.166052+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]': finished 2024-07-28T19:43:39.578 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:39 smithi100 bash[82644]: cluster 2024-07-28T19:43:38.166174+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-07-28T19:43:39.822 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:39.822 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:39.822 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:39.823 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:39.823 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:39.823 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:39.823 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:finished writesame 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:to verify the data 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:verified 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:started writesame 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:expected fail, finished writesame 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (7661 ms) 2024-07-28T19:43:40.229 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-07-28T19:43:40.230 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:40.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:40 smithi100 bash[80199]: cluster 2024-07-28T19:43:39.161855+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-07-28T19:43:40.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:40 smithi100 bash[82644]: cluster 2024-07-28T19:43:39.161855+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-07-28T19:43:40.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:40 smithi138 bash[67894]: cluster 2024-07-28T19:43:39.161855+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-07-28T19:43:41.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:41 smithi100 bash[80199]: cluster 2024-07-28T19:43:39.819241+0000 mgr.y (mgr.25358) 332 : cluster [DBG] pgmap v602: 457 pgs: 3 peering, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.53k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:41.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:41 smithi100 bash[82644]: cluster 2024-07-28T19:43:39.819241+0000 mgr.y (mgr.25358) 332 : cluster [DBG] pgmap v602: 457 pgs: 3 peering, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.53k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:41 smithi138 bash[67894]: cluster 2024-07-28T19:43:39.819241+0000 mgr.y (mgr.25358) 332 : cluster [DBG] pgmap v602: 457 pgs: 3 peering, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.53k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 0 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 1 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:snap: orig 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:found orig with size 2097152 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:read: 80 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:write test data! 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 80 2024-07-28T19:43:42.278 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 2 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:snap: orig 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:snap: written 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:found orig with size 2097152 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:found written with size 2097152 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:read: 80 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:read: 80 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:read: 80 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:write to snapshot returned -30 2024-07-28T19:43:42.279 INFO:tasks.workunit.client.0.smithi100.stdout:Read-only file system 2024-07-28T19:43:42.331 INFO:tasks.workunit.client.0.smithi100.stdout:read: 80 2024-07-28T19:43:42.331 INFO:tasks.workunit.client.0.smithi100.stdout:read: 80 2024-07-28T19:43:42.331 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 80 2024-07-28T19:43:42.331 INFO:tasks.workunit.client.0.smithi100.stdout:opening testimg@orig 2024-07-28T19:43:42.331 INFO:tasks.workunit.client.0.smithi100.stdout:read: 80 2024-07-28T19:43:42.331 INFO:tasks.workunit.client.0.smithi100.stdout:write to snapshot returned -30 2024-07-28T19:43:42.331 INFO:tasks.workunit.client.0.smithi100.stdout:Read-only file system 2024-07-28T19:43:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:42 smithi100 bash[80199]: cluster 2024-07-28T19:43:41.196651+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-07-28T19:43:42.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:42 smithi100 bash[82644]: cluster 2024-07-28T19:43:41.196651+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-07-28T19:43:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:42 smithi138 bash[67894]: cluster 2024-07-28T19:43:41.196651+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-07-28T19:43:42.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:43:42 smithi138 bash[60615]: ts=2024-07-28T19:43:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:43:43.202 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:43:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:43:42] "GET /metrics HTTP/1.1" 200 63171 "" "Prometheus/2.43.0" 2024-07-28T19:43:43.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:43 smithi100 bash[80199]: cluster 2024-07-28T19:43:41.820353+0000 mgr.y (mgr.25358) 333 : cluster [DBG] pgmap v604: 457 pgs: 3 peering, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 448 KiB/s rd, 224 KiB/s wr, 920 op/s; 2 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:43:43.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:43 smithi100 bash[80199]: cluster 2024-07-28T19:43:42.207613+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-07-28T19:43:43.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:43 smithi100 bash[82644]: cluster 2024-07-28T19:43:41.820353+0000 mgr.y (mgr.25358) 333 : cluster [DBG] pgmap v604: 457 pgs: 3 peering, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 448 KiB/s rd, 224 KiB/s wr, 920 op/s; 2 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:43:43.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:43 smithi100 bash[82644]: cluster 2024-07-28T19:43:42.207613+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-07-28T19:43:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:43 smithi138 bash[67894]: cluster 2024-07-28T19:43:41.820353+0000 mgr.y (mgr.25358) 333 : cluster [DBG] pgmap v604: 457 pgs: 3 peering, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 448 KiB/s rd, 224 KiB/s wr, 920 op/s; 2 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:43:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:43 smithi138 bash[67894]: cluster 2024-07-28T19:43:42.207613+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-07-28T19:43:44.262 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 2 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:snap: orig 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:snap: written 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:found orig with size 2097152 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:found written with size 2097152 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 1 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:snap: orig 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:found orig with size 2097152 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 0 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (4034 ms) 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-07-28T19:43:44.263 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:44.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:44 smithi100 bash[80199]: cluster 2024-07-28T19:43:43.217509+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-07-28T19:43:44.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:44 smithi100 bash[82644]: cluster 2024-07-28T19:43:43.217509+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-07-28T19:43:44.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:44 smithi138 bash[67894]: cluster 2024-07-28T19:43:43.217509+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-07-28T19:43:45.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:45 smithi100 bash[80199]: cluster 2024-07-28T19:43:43.822828+0000 mgr.y (mgr.25358) 334 : cluster [DBG] pgmap v607: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 185 KiB/s rd, 87 KiB/s wr, 364 op/s; 7 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:43:45.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:45 smithi100 bash[80199]: cluster 2024-07-28T19:43:44.227914+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-07-28T19:43:45.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:45 smithi100 bash[82644]: cluster 2024-07-28T19:43:43.822828+0000 mgr.y (mgr.25358) 334 : cluster [DBG] pgmap v607: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 185 KiB/s rd, 87 KiB/s wr, 364 op/s; 7 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:43:45.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:45 smithi100 bash[82644]: cluster 2024-07-28T19:43:44.227914+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-07-28T19:43:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:45 smithi138 bash[67894]: cluster 2024-07-28T19:43:43.822828+0000 mgr.y (mgr.25358) 334 : cluster [DBG] pgmap v607: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 185 KiB/s rd, 87 KiB/s wr, 364 op/s; 7 B/s, 1 keys/s, 0 objects/s recovering 2024-07-28T19:43:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:45 smithi138 bash[67894]: cluster 2024-07-28T19:43:44.227914+0000 mon.a (mon.0) 1061 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-07-28T19:43:46.303 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (2040 ms) 2024-07-28T19:43:46.303 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestClone 2024-07-28T19:43:46.303 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:46.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:46 smithi138 bash[67894]: cluster 2024-07-28T19:43:45.231850+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-07-28T19:43:46.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:46 smithi100 bash[80199]: cluster 2024-07-28T19:43:45.231850+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-07-28T19:43:46.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:46 smithi100 bash[82644]: cluster 2024-07-28T19:43:45.231850+0000 mon.a (mon.0) 1062 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-07-28T19:43:47.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:47 smithi138 bash[67894]: cluster 2024-07-28T19:43:45.823813+0000 mgr.y (mgr.25358) 335 : cluster [DBG] pgmap v610: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 226 KiB/s rd, 104 KiB/s wr, 444 op/s; 4 B/s, 0 objects/s recovering 2024-07-28T19:43:47.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:47 smithi138 bash[67894]: cluster 2024-07-28T19:43:46.257462+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-07-28T19:43:47.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:47 smithi100 bash[82644]: cluster 2024-07-28T19:43:45.823813+0000 mgr.y (mgr.25358) 335 : cluster [DBG] pgmap v610: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 226 KiB/s rd, 104 KiB/s wr, 444 op/s; 4 B/s, 0 objects/s recovering 2024-07-28T19:43:47.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:47 smithi100 bash[82644]: cluster 2024-07-28T19:43:46.257462+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-07-28T19:43:47.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:47 smithi100 bash[80199]: cluster 2024-07-28T19:43:45.823813+0000 mgr.y (mgr.25358) 335 : cluster [DBG] pgmap v610: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 226 KiB/s rd, 104 KiB/s wr, 444 op/s; 4 B/s, 0 objects/s recovering 2024-07-28T19:43:47.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:47 smithi100 bash[80199]: cluster 2024-07-28T19:43:46.257462+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-07-28T19:43:48.313 INFO:tasks.workunit.client.0.smithi100.stdout:made parent image "parent" 2024-07-28T19:43:48.313 INFO:tasks.workunit.client.0.smithi100.stdout:parent has no parent info 2024-07-28T19:43:48.313 INFO:tasks.workunit.client.0.smithi100.stdout:made snapshot "parent@parent_snap" 2024-07-28T19:43:48.313 INFO:tasks.workunit.client.0.smithi100.stdout:can't unprotect an unprotected snap 2024-07-28T19:43:48.313 INFO:tasks.workunit.client.0.smithi100.stdout:can't protect a protected snap 2024-07-28T19:43:48.313 INFO:tasks.workunit.client.0.smithi100.stdout:made and opened clone "child" 2024-07-28T19:43:48.313 INFO:tasks.workunit.client.0.smithi100.stdout:read: 8 2024-07-28T19:43:48.313 INFO:tasks.workunit.client.0.smithi100.stdout:read: 8 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:read: 8 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:sizes and overlaps are good between parent and child 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:child image successfully cloned all image-meta pairs 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:sized down clone, changed overlap 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:sized up clone, changed size but not overlap or parent's size 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:can't remove parent while child still exists 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:can't remove parent while still protected 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:removed parent snap after unprotecting 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestClone (2010 ms) 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestClone2 2024-07-28T19:43:48.314 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:48.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:48 smithi138 bash[67894]: cluster 2024-07-28T19:43:47.270123+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-07-28T19:43:48.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:48 smithi100 bash[80199]: cluster 2024-07-28T19:43:47.270123+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-07-28T19:43:48.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:48 smithi100 bash[82644]: cluster 2024-07-28T19:43:47.270123+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-07-28T19:43:49.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:49 smithi138 bash[67894]: cluster 2024-07-28T19:43:47.824818+0000 mgr.y (mgr.25358) 336 : cluster [DBG] pgmap v613: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 85 op/s 2024-07-28T19:43:49.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:49 smithi138 bash[67894]: cluster 2024-07-28T19:43:48.283799+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-07-28T19:43:49.616 INFO:tasks.workunit.client.0.smithi100.stdout:made parent image "parent" 2024-07-28T19:43:49.616 INFO:tasks.workunit.client.0.smithi100.stdout:parent has no parent info 2024-07-28T19:43:49.616 INFO:tasks.workunit.client.0.smithi100.stdout:made snapshot "parent@parent_snap" 2024-07-28T19:43:49.616 INFO:tasks.workunit.client.0.smithi100.stdout:made and opened clone "child" 2024-07-28T19:43:49.617 INFO:tasks.workunit.client.0.smithi100.stdout:child image successfully cloned all image-meta pairs 2024-07-28T19:43:49.617 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestClone2 (1303 ms) 2024-07-28T19:43:49.617 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ListChildren 2024-07-28T19:43:49.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:49 smithi100 bash[80199]: cluster 2024-07-28T19:43:47.824818+0000 mgr.y (mgr.25358) 336 : cluster [DBG] pgmap v613: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 85 op/s 2024-07-28T19:43:49.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:49 smithi100 bash[80199]: cluster 2024-07-28T19:43:48.283799+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-07-28T19:43:49.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:49 smithi100 bash[82644]: cluster 2024-07-28T19:43:47.824818+0000 mgr.y (mgr.25358) 336 : cluster [DBG] pgmap v613: 457 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 454 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 85 op/s 2024-07-28T19:43:49.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:49 smithi100 bash[82644]: cluster 2024-07-28T19:43:48.283799+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-07-28T19:43:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:50 smithi138 bash[67894]: cluster 2024-07-28T19:43:49.288250+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-07-28T19:43:50.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:50 smithi100 bash[80199]: cluster 2024-07-28T19:43:49.288250+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-07-28T19:43:50.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:50 smithi100 bash[82644]: cluster 2024-07-28T19:43:49.288250+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-07-28T19:43:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:51 smithi138 bash[67894]: cluster 2024-07-28T19:43:49.826324+0000 mgr.y (mgr.25358) 337 : cluster [DBG] pgmap v616: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 743 KiB/s rd, 53 KiB/s wr, 891 op/s 2024-07-28T19:43:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:51 smithi138 bash[67894]: cluster 2024-07-28T19:43:50.302708+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-07-28T19:43:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:51 smithi138 bash[67894]: audit 2024-07-28T19:43:50.307845+0000 mon.a (mon.0) 1068 : audit [INF] from='client.? 172.21.15.100:0/2290650310' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:51.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:51 smithi100 bash[80199]: cluster 2024-07-28T19:43:49.826324+0000 mgr.y (mgr.25358) 337 : cluster [DBG] pgmap v616: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 743 KiB/s rd, 53 KiB/s wr, 891 op/s 2024-07-28T19:43:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:51 smithi100 bash[80199]: cluster 2024-07-28T19:43:50.302708+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-07-28T19:43:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:51 smithi100 bash[80199]: audit 2024-07-28T19:43:50.307845+0000 mon.a (mon.0) 1068 : audit [INF] from='client.? 172.21.15.100:0/2290650310' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:51 smithi100 bash[82644]: cluster 2024-07-28T19:43:49.826324+0000 mgr.y (mgr.25358) 337 : cluster [DBG] pgmap v616: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 743 KiB/s rd, 53 KiB/s wr, 891 op/s 2024-07-28T19:43:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:51 smithi100 bash[82644]: cluster 2024-07-28T19:43:50.302708+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-07-28T19:43:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:51 smithi100 bash[82644]: audit 2024-07-28T19:43:50.307845+0000 mon.a (mon.0) 1068 : audit [INF] from='client.? 172.21.15.100:0/2290650310' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:52.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:52 smithi100 bash[80199]: audit 2024-07-28T19:43:51.311369+0000 mon.a (mon.0) 1069 : audit [INF] from='client.? 172.21.15.100:0/2290650310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:52 smithi100 bash[80199]: cluster 2024-07-28T19:43:51.311980+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-07-28T19:43:52.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:52 smithi100 bash[82644]: audit 2024-07-28T19:43:51.311369+0000 mon.a (mon.0) 1069 : audit [INF] from='client.? 172.21.15.100:0/2290650310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:52.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:52 smithi100 bash[82644]: cluster 2024-07-28T19:43:51.311980+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-07-28T19:43:52.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:43:52 smithi138 bash[60615]: ts=2024-07-28T19:43:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:43:52.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:52 smithi138 bash[67894]: audit 2024-07-28T19:43:51.311369+0000 mon.a (mon.0) 1069 : audit [INF] from='client.? 172.21.15.100:0/2290650310' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:52.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:52 smithi138 bash[67894]: cluster 2024-07-28T19:43:51.311980+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-07-28T19:43:53.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:43:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:43:52] "GET /metrics HTTP/1.1" 200 67912 "" "Prometheus/2.43.0" 2024-07-28T19:43:53.360 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:43:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[82644]: cluster 2024-07-28T19:43:51.827314+0000 mgr.y (mgr.25358) 338 : cluster [DBG] pgmap v619: 489 pgs: 32 unknown, 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 691 KiB/s rd, 51 KiB/s wr, 831 op/s 2024-07-28T19:43:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[82644]: cluster 2024-07-28T19:43:52.323083+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-07-28T19:43:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[82644]: audit 2024-07-28T19:43:52.325111+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.100:0/3706627224' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[82644]: audit 2024-07-28T19:43:52.326122+0000 mon.a (mon.0) 1072 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[82644]: audit 2024-07-28T19:43:52.636911+0000 mon.a (mon.0) 1073 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[82644]: audit 2024-07-28T19:43:53.126784+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[80199]: cluster 2024-07-28T19:43:51.827314+0000 mgr.y (mgr.25358) 338 : cluster [DBG] pgmap v619: 489 pgs: 32 unknown, 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 691 KiB/s rd, 51 KiB/s wr, 831 op/s 2024-07-28T19:43:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[80199]: cluster 2024-07-28T19:43:52.323083+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-07-28T19:43:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[80199]: audit 2024-07-28T19:43:52.325111+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.100:0/3706627224' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[80199]: audit 2024-07-28T19:43:52.326122+0000 mon.a (mon.0) 1072 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[80199]: audit 2024-07-28T19:43:52.636911+0000 mon.a (mon.0) 1073 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:53 smithi100 bash[80199]: audit 2024-07-28T19:43:53.126784+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:53 smithi138 bash[67894]: cluster 2024-07-28T19:43:51.827314+0000 mgr.y (mgr.25358) 338 : cluster [DBG] pgmap v619: 489 pgs: 32 unknown, 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 691 KiB/s rd, 51 KiB/s wr, 831 op/s 2024-07-28T19:43:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:53 smithi138 bash[67894]: cluster 2024-07-28T19:43:52.323083+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-07-28T19:43:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:53 smithi138 bash[67894]: audit 2024-07-28T19:43:52.325111+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.100:0/3706627224' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:53 smithi138 bash[67894]: audit 2024-07-28T19:43:52.326122+0000 mon.a (mon.0) 1072 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:43:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:53 smithi138 bash[67894]: audit 2024-07-28T19:43:52.636911+0000 mon.a (mon.0) 1073 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:43:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:53 smithi138 bash[67894]: audit 2024-07-28T19:43:53.126784+0000 mon.a (mon.0) 1074 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:43:54.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:54 smithi100 bash[80199]: audit 2024-07-28T19:43:53.325574+0000 mon.a (mon.0) 1075 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:54.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:54 smithi100 bash[80199]: cluster 2024-07-28T19:43:53.326086+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-07-28T19:43:54.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:54 smithi100 bash[82644]: audit 2024-07-28T19:43:53.325574+0000 mon.a (mon.0) 1075 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:54.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:54 smithi100 bash[82644]: cluster 2024-07-28T19:43:53.326086+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-07-28T19:43:54.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:54 smithi138 bash[67894]: audit 2024-07-28T19:43:53.325574+0000 mon.a (mon.0) 1075 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:43:54.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:54 smithi138 bash[67894]: cluster 2024-07-28T19:43:53.326086+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-07-28T19:43:55.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:55 smithi100 bash[80199]: cluster 2024-07-28T19:43:53.829931+0000 mgr.y (mgr.25358) 339 : cluster [DBG] pgmap v622: 521 pgs: 30 unknown, 491 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.5 KiB/s wr, 76 op/s 2024-07-28T19:43:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:55 smithi100 bash[80199]: cluster 2024-07-28T19:43:54.365362+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-07-28T19:43:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:55 smithi100 bash[80199]: cluster 2024-07-28T19:43:55.145437+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-07-28T19:43:55.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:55 smithi100 bash[82644]: cluster 2024-07-28T19:43:53.829931+0000 mgr.y (mgr.25358) 339 : cluster [DBG] pgmap v622: 521 pgs: 30 unknown, 491 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.5 KiB/s wr, 76 op/s 2024-07-28T19:43:55.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:55 smithi100 bash[82644]: cluster 2024-07-28T19:43:54.365362+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-07-28T19:43:55.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:55 smithi100 bash[82644]: cluster 2024-07-28T19:43:55.145437+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-07-28T19:43:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:55 smithi138 bash[67894]: cluster 2024-07-28T19:43:53.829931+0000 mgr.y (mgr.25358) 339 : cluster [DBG] pgmap v622: 521 pgs: 30 unknown, 491 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.5 KiB/s wr, 76 op/s 2024-07-28T19:43:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:55 smithi138 bash[67894]: cluster 2024-07-28T19:43:54.365362+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-07-28T19:43:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:55 smithi138 bash[67894]: cluster 2024-07-28T19:43:55.145437+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-07-28T19:43:57.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:57 smithi100 bash[80199]: cluster 2024-07-28T19:43:55.831049+0000 mgr.y (mgr.25358) 340 : cluster [DBG] pgmap v625: 521 pgs: 4 unknown, 517 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 5.5 KiB/s wr, 99 op/s 2024-07-28T19:43:57.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:57 smithi100 bash[80199]: cluster 2024-07-28T19:43:56.144757+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-07-28T19:43:57.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:57 smithi100 bash[82644]: cluster 2024-07-28T19:43:55.831049+0000 mgr.y (mgr.25358) 340 : cluster [DBG] pgmap v625: 521 pgs: 4 unknown, 517 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 5.5 KiB/s wr, 99 op/s 2024-07-28T19:43:57.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:57 smithi100 bash[82644]: cluster 2024-07-28T19:43:56.144757+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-07-28T19:43:57.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:57 smithi138 bash[67894]: cluster 2024-07-28T19:43:55.831049+0000 mgr.y (mgr.25358) 340 : cluster [DBG] pgmap v625: 521 pgs: 4 unknown, 517 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 5.5 KiB/s wr, 99 op/s 2024-07-28T19:43:57.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:57 smithi138 bash[67894]: cluster 2024-07-28T19:43:56.144757+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-07-28T19:43:58.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:58 smithi100 bash[80199]: cluster 2024-07-28T19:43:57.156473+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-07-28T19:43:58.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:58 smithi100 bash[82644]: cluster 2024-07-28T19:43:57.156473+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-07-28T19:43:58.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:58 smithi138 bash[67894]: cluster 2024-07-28T19:43:57.156473+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-07-28T19:43:59.479 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.479 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.479 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.479 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.479 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.479 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 1 2024-07-28T19:43:59.479 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:child: image96 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96/85fba56ed582 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.480 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 2 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:child: image97 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:child: image96 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96/85fba56ed582 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97/85fb4a9e1099 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.481 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image98 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image98 2024-07-28T19:43:59.482 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image98 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 3 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:child: image97 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:child: image96 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:child: image98 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96/85fba56ed582 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97/85fb4a9e1099 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image98/85fbb2a368ab 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.483 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 3 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:child: image97 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:child: image96 2024-07-28T19:43:59.484 INFO:tasks.workunit.client.0.smithi100.stdout:child: image98 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96/85fba56ed582 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97/85fb4a9e1099 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image98/85fbb2a368ab 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.485 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image99 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image99 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image99 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 4 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 4 2024-07-28T19:43:59.486 INFO:tasks.workunit.client.0.smithi100.stdout:child: image97 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout:child: image96 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout:child: image98 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout:child: image99 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96/85fba56ed582 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97/85fb4a9e1099 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image98/85fbb2a368ab 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image99/85fbcb7e9cb5 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.487 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image98 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image98 2024-07-28T19:43:59.488 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image98 2024-07-28T19:43:59.489 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.489 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:43:59.489 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image99 2024-07-28T19:43:59.489 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:43:59.489 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image96 2024-07-28T19:43:59.489 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image98 2024-07-28T19:43:59.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:59 smithi100 bash[82644]: cluster 2024-07-28T19:43:57.832844+0000 mgr.y (mgr.25358) 341 : cluster [DBG] pgmap v628: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 KiB/s wr, 33 op/s 2024-07-28T19:43:59.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:43:59 smithi100 bash[82644]: cluster 2024-07-28T19:43:58.168579+0000 mon.a (mon.0) 1081 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-07-28T19:43:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:59 smithi100 bash[80199]: cluster 2024-07-28T19:43:57.832844+0000 mgr.y (mgr.25358) 341 : cluster [DBG] pgmap v628: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 KiB/s wr, 33 op/s 2024-07-28T19:43:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:43:59 smithi100 bash[80199]: cluster 2024-07-28T19:43:58.168579+0000 mon.a (mon.0) 1081 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-07-28T19:43:59.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:59 smithi138 bash[67894]: cluster 2024-07-28T19:43:57.832844+0000 mgr.y (mgr.25358) 341 : cluster [DBG] pgmap v628: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 KiB/s wr, 33 op/s 2024-07-28T19:43:59.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:43:59 smithi138 bash[67894]: cluster 2024-07-28T19:43:58.168579+0000 mon.a (mon.0) 1081 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-07-28T19:44:00.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:00 smithi100 bash[80199]: audit 2024-07-28T19:43:59.680594+0000 mon.a (mon.0) 1082 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:44:00.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:00 smithi100 bash[82644]: audit 2024-07-28T19:43:59.680594+0000 mon.a (mon.0) 1082 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:44:00.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:00 smithi138 bash[67894]: audit 2024-07-28T19:43:59.680594+0000 mon.a (mon.0) 1082 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image99 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image99 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 4 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 4 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:child: image97 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:child: image96 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:child: image98 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:child: image99 2024-07-28T19:44:01.419 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image96/85fba56ed582 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97/85fb4a9e1099 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image98/85fbb2a368ab 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image99/85fbcb7e9cb5 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.420 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image98 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image98 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image98 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image99 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image98 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image99 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image99 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 3 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:child: image97 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:child: image98 2024-07-28T19:44:01.421 INFO:tasks.workunit.client.0.smithi100.stdout:child: image99 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97/85fb4a9e1099 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image98/85fbb2a368ab 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image99/85fbcb7e9cb5 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.422 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-27/image99 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-27/image99 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image99 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 2 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:child: image97 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:child: image99 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97/85fb4a9e1099 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-27/image99/85fbcb7e9cb5 2024-07-28T19:44:01.423 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 1 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:child: image97 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-26/image97/85fb4a9e1099 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 0 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ListChildren (11803 ms) 2024-07-28T19:44:01.424 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-07-28T19:44:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:01 smithi100 bash[80199]: cluster 2024-07-28T19:43:59.836673+0000 mgr.y (mgr.25358) 342 : cluster [DBG] pgmap v630: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 4.5 KiB/s wr, 119 op/s 2024-07-28T19:44:01.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:01 smithi100 bash[82644]: cluster 2024-07-28T19:43:59.836673+0000 mgr.y (mgr.25358) 342 : cluster [DBG] pgmap v630: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 4.5 KiB/s wr, 119 op/s 2024-07-28T19:44:01.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:01 smithi138 bash[67894]: cluster 2024-07-28T19:43:59.836673+0000 mgr.y (mgr.25358) 342 : cluster [DBG] pgmap v630: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 4.5 KiB/s wr, 119 op/s 2024-07-28T19:44:02.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:02 smithi100 bash[80199]: cluster 2024-07-28T19:44:01.185610+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-07-28T19:44:02.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:02 smithi100 bash[82644]: cluster 2024-07-28T19:44:01.185610+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-07-28T19:44:02.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:02 smithi138 bash[67894]: cluster 2024-07-28T19:44:01.185610+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-07-28T19:44:02.556 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:44:02 smithi138 bash[60615]: ts=2024-07-28T19:44:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:44:03.214 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:44:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:44:02] "GET /metrics HTTP/1.1" 200 67912 "" "Prometheus/2.43.0" 2024-07-28T19:44:03.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[80199]: cluster 2024-07-28T19:44:01.837801+0000 mgr.y (mgr.25358) 343 : cluster [DBG] pgmap v632: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 232 KiB/s rd, 6.0 KiB/s wr, 278 op/s 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[80199]: cluster 2024-07-28T19:44:02.197880+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[80199]: audit 2024-07-28T19:44:02.201003+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.100:0/1051517868' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[80199]: audit 2024-07-28T19:44:02.201930+0000 mon.a (mon.0) 1085 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[80199]: audit 2024-07-28T19:44:03.187531+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[82644]: cluster 2024-07-28T19:44:01.837801+0000 mgr.y (mgr.25358) 343 : cluster [DBG] pgmap v632: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 232 KiB/s rd, 6.0 KiB/s wr, 278 op/s 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[82644]: cluster 2024-07-28T19:44:02.197880+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[82644]: audit 2024-07-28T19:44:02.201003+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.100:0/1051517868' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[82644]: audit 2024-07-28T19:44:02.201930+0000 mon.a (mon.0) 1085 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:03 smithi100 bash[82644]: audit 2024-07-28T19:44:03.187531+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:03 smithi138 bash[67894]: cluster 2024-07-28T19:44:01.837801+0000 mgr.y (mgr.25358) 343 : cluster [DBG] pgmap v632: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 232 KiB/s rd, 6.0 KiB/s wr, 278 op/s 2024-07-28T19:44:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:03 smithi138 bash[67894]: cluster 2024-07-28T19:44:02.197880+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-07-28T19:44:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:03 smithi138 bash[67894]: audit 2024-07-28T19:44:02.201003+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.100:0/1051517868' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:03 smithi138 bash[67894]: audit 2024-07-28T19:44:02.201930+0000 mon.a (mon.0) 1085 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:03 smithi138 bash[67894]: audit 2024-07-28T19:44:03.187531+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:04.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:04 smithi100 bash[80199]: audit 2024-07-28T19:44:03.203487+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:04.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:04 smithi100 bash[80199]: cluster 2024-07-28T19:44:03.204243+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-07-28T19:44:04.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:04 smithi100 bash[82644]: audit 2024-07-28T19:44:03.203487+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:04.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:04 smithi100 bash[82644]: cluster 2024-07-28T19:44:03.204243+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-07-28T19:44:04.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:04 smithi138 bash[67894]: audit 2024-07-28T19:44:03.203487+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:04.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:04 smithi138 bash[67894]: cluster 2024-07-28T19:44:03.204243+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-07-28T19:44:05.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[82644]: cluster 2024-07-28T19:44:03.840565+0000 mgr.y (mgr.25358) 344 : cluster [DBG] pgmap v635: 553 pgs: 6 creating+peering, 13 unknown, 534 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 367 KiB/s rd, 7.9 KiB/s wr, 446 op/s 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[82644]: cluster 2024-07-28T19:44:04.226316+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[82644]: audit 2024-07-28T19:44:04.228396+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.100:0/1075847301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[82644]: audit 2024-07-28T19:44:04.242019+0000 mon.a (mon.0) 1090 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[82644]: audit 2024-07-28T19:44:05.143380+0000 mon.a (mon.0) 1091 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[82644]: cluster 2024-07-28T19:44:05.143834+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[80199]: cluster 2024-07-28T19:44:03.840565+0000 mgr.y (mgr.25358) 344 : cluster [DBG] pgmap v635: 553 pgs: 6 creating+peering, 13 unknown, 534 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 367 KiB/s rd, 7.9 KiB/s wr, 446 op/s 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[80199]: cluster 2024-07-28T19:44:04.226316+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[80199]: audit 2024-07-28T19:44:04.228396+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.100:0/1075847301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[80199]: audit 2024-07-28T19:44:04.242019+0000 mon.a (mon.0) 1090 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:05.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[80199]: audit 2024-07-28T19:44:05.143380+0000 mon.a (mon.0) 1091 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:06.000 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:05 smithi100 bash[80199]: cluster 2024-07-28T19:44:05.143834+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-07-28T19:44:06.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:05 smithi138 bash[67894]: cluster 2024-07-28T19:44:03.840565+0000 mgr.y (mgr.25358) 344 : cluster [DBG] pgmap v635: 553 pgs: 6 creating+peering, 13 unknown, 534 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 367 KiB/s rd, 7.9 KiB/s wr, 446 op/s 2024-07-28T19:44:06.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:05 smithi138 bash[67894]: cluster 2024-07-28T19:44:04.226316+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-07-28T19:44:06.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:05 smithi138 bash[67894]: audit 2024-07-28T19:44:04.228396+0000 mon.c (mon.2) 20 : audit [INF] from='client.? 172.21.15.100:0/1075847301' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:06.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:05 smithi138 bash[67894]: audit 2024-07-28T19:44:04.242019+0000 mon.a (mon.0) 1090 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:06.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:05 smithi138 bash[67894]: audit 2024-07-28T19:44:05.143380+0000 mon.a (mon.0) 1091 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:06.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:05 smithi138 bash[67894]: cluster 2024-07-28T19:44:05.143834+0000 mon.a (mon.0) 1092 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-07-28T19:44:06.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: audit 2024-07-28T19:44:05.616306+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: audit 2024-07-28T19:44:05.622545+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: audit 2024-07-28T19:44:05.631495+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: audit 2024-07-28T19:44:05.638905+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: cluster 2024-07-28T19:44:05.841641+0000 mgr.y (mgr.25358) 345 : cluster [DBG] pgmap v638: 585 pgs: 9 creating+peering, 29 unknown, 547 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 264 KiB/s rd, 7.7 KiB/s wr, 333 op/s 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: cluster 2024-07-28T19:44:06.162575+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: audit 2024-07-28T19:44:06.165197+0000 mon.a (mon.0) 1098 : audit [INF] from='client.? 172.21.15.100:0/2821196121' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: audit 2024-07-28T19:44:06.187367+0000 mon.a (mon.0) 1099 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: audit 2024-07-28T19:44:06.190913+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[80199]: audit 2024-07-28T19:44:06.207456+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: audit 2024-07-28T19:44:05.616306+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: audit 2024-07-28T19:44:05.622545+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: audit 2024-07-28T19:44:05.631495+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: audit 2024-07-28T19:44:05.638905+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: cluster 2024-07-28T19:44:05.841641+0000 mgr.y (mgr.25358) 345 : cluster [DBG] pgmap v638: 585 pgs: 9 creating+peering, 29 unknown, 547 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 264 KiB/s rd, 7.7 KiB/s wr, 333 op/s 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: cluster 2024-07-28T19:44:06.162575+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-07-28T19:44:06.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: audit 2024-07-28T19:44:06.165197+0000 mon.a (mon.0) 1098 : audit [INF] from='client.? 172.21.15.100:0/2821196121' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:07.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: audit 2024-07-28T19:44:06.187367+0000 mon.a (mon.0) 1099 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:44:07.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: audit 2024-07-28T19:44:06.190913+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:44:07.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:06 smithi100 bash[82644]: audit 2024-07-28T19:44:06.207456+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: audit 2024-07-28T19:44:05.616306+0000 mon.a (mon.0) 1093 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: audit 2024-07-28T19:44:05.622545+0000 mon.a (mon.0) 1094 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: audit 2024-07-28T19:44:05.631495+0000 mon.a (mon.0) 1095 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: audit 2024-07-28T19:44:05.638905+0000 mon.a (mon.0) 1096 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: cluster 2024-07-28T19:44:05.841641+0000 mgr.y (mgr.25358) 345 : cluster [DBG] pgmap v638: 585 pgs: 9 creating+peering, 29 unknown, 547 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 264 KiB/s rd, 7.7 KiB/s wr, 333 op/s 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: cluster 2024-07-28T19:44:06.162575+0000 mon.a (mon.0) 1097 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: audit 2024-07-28T19:44:06.165197+0000 mon.a (mon.0) 1098 : audit [INF] from='client.? 172.21.15.100:0/2821196121' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: audit 2024-07-28T19:44:06.187367+0000 mon.a (mon.0) 1099 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: audit 2024-07-28T19:44:06.190913+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:44:07.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:06 smithi138 bash[67894]: audit 2024-07-28T19:44:06.207456+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:08.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:08 smithi100 bash[80199]: audit 2024-07-28T19:44:07.171041+0000 mon.a (mon.0) 1102 : audit [INF] from='client.? 172.21.15.100:0/2821196121' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:08.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:08 smithi100 bash[80199]: cluster 2024-07-28T19:44:07.171711+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-07-28T19:44:08.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:08 smithi100 bash[80199]: audit 2024-07-28T19:44:07.184256+0000 mon.a (mon.0) 1104 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30", "force_nonempty":""}]: dispatch 2024-07-28T19:44:08.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:08 smithi100 bash[80199]: audit 2024-07-28T19:44:07.637454+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:08.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:08 smithi100 bash[82644]: audit 2024-07-28T19:44:07.171041+0000 mon.a (mon.0) 1102 : audit [INF] from='client.? 172.21.15.100:0/2821196121' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:08.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:08 smithi100 bash[82644]: cluster 2024-07-28T19:44:07.171711+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-07-28T19:44:08.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:08 smithi100 bash[82644]: audit 2024-07-28T19:44:07.184256+0000 mon.a (mon.0) 1104 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30", "force_nonempty":""}]: dispatch 2024-07-28T19:44:08.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:08 smithi100 bash[82644]: audit 2024-07-28T19:44:07.637454+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:08 smithi138 bash[67894]: audit 2024-07-28T19:44:07.171041+0000 mon.a (mon.0) 1102 : audit [INF] from='client.? 172.21.15.100:0/2821196121' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:44:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:08 smithi138 bash[67894]: cluster 2024-07-28T19:44:07.171711+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-07-28T19:44:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:08 smithi138 bash[67894]: audit 2024-07-28T19:44:07.184256+0000 mon.a (mon.0) 1104 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30", "force_nonempty":""}]: dispatch 2024-07-28T19:44:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:08 smithi138 bash[67894]: audit 2024-07-28T19:44:07.637454+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:09.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:09 smithi100 bash[80199]: cluster 2024-07-28T19:44:07.843525+0000 mgr.y (mgr.25358) 346 : cluster [DBG] pgmap v641: 617 pgs: 3 creating+peering, 49 unknown, 565 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 4.5 KiB/s wr, 92 op/s 2024-07-28T19:44:09.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:09 smithi100 bash[80199]: audit 2024-07-28T19:44:08.177301+0000 mon.a (mon.0) 1106 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30", "force_nonempty":""}]': finished 2024-07-28T19:44:09.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:09 smithi100 bash[80199]: cluster 2024-07-28T19:44:08.177460+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-07-28T19:44:09.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:09 smithi100 bash[80199]: audit 2024-07-28T19:44:08.179326+0000 mon.a (mon.0) 1108 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi100-68802-30", "mode":"writeback"}]: dispatch 2024-07-28T19:44:09.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:09 smithi100 bash[82644]: cluster 2024-07-28T19:44:07.843525+0000 mgr.y (mgr.25358) 346 : cluster [DBG] pgmap v641: 617 pgs: 3 creating+peering, 49 unknown, 565 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 4.5 KiB/s wr, 92 op/s 2024-07-28T19:44:09.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:09 smithi100 bash[82644]: audit 2024-07-28T19:44:08.177301+0000 mon.a (mon.0) 1106 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30", "force_nonempty":""}]': finished 2024-07-28T19:44:09.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:09 smithi100 bash[82644]: cluster 2024-07-28T19:44:08.177460+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-07-28T19:44:09.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:09 smithi100 bash[82644]: audit 2024-07-28T19:44:08.179326+0000 mon.a (mon.0) 1108 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi100-68802-30", "mode":"writeback"}]: dispatch 2024-07-28T19:44:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:09 smithi138 bash[67894]: cluster 2024-07-28T19:44:07.843525+0000 mgr.y (mgr.25358) 346 : cluster [DBG] pgmap v641: 617 pgs: 3 creating+peering, 49 unknown, 565 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 4.5 KiB/s wr, 92 op/s 2024-07-28T19:44:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:09 smithi138 bash[67894]: audit 2024-07-28T19:44:08.177301+0000 mon.a (mon.0) 1106 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30", "force_nonempty":""}]': finished 2024-07-28T19:44:09.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:09 smithi138 bash[67894]: cluster 2024-07-28T19:44:08.177460+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-07-28T19:44:09.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:09 smithi138 bash[67894]: audit 2024-07-28T19:44:08.179326+0000 mon.a (mon.0) 1108 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi100-68802-30", "mode":"writeback"}]: dispatch 2024-07-28T19:44:10.148 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:10.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[80199]: cluster 2024-07-28T19:44:09.169459+0000 mon.a (mon.0) 1109 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-07-28T19:44:10.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[80199]: audit 2024-07-28T19:44:09.177919+0000 mon.a (mon.0) 1110 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi100-68802-30", "mode":"writeback"}]': finished 2024-07-28T19:44:10.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[80199]: cluster 2024-07-28T19:44:09.178072+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-07-28T19:44:10.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[80199]: audit 2024-07-28T19:44:09.180141+0000 mon.a (mon.0) 1112 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi100-68802-28", "overlaypool":"test-librbd-smithi100-68802-30"}]: dispatch 2024-07-28T19:44:10.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[80199]: audit 2024-07-28T19:44:10.148114+0000 mon.a (mon.0) 1113 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi100-68802-28", "overlaypool":"test-librbd-smithi100-68802-30"}]': finished 2024-07-28T19:44:10.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[80199]: cluster 2024-07-28T19:44:10.148262+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-07-28T19:44:10.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[82644]: cluster 2024-07-28T19:44:09.169459+0000 mon.a (mon.0) 1109 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-07-28T19:44:10.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[82644]: audit 2024-07-28T19:44:09.177919+0000 mon.a (mon.0) 1110 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi100-68802-30", "mode":"writeback"}]': finished 2024-07-28T19:44:10.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[82644]: cluster 2024-07-28T19:44:09.178072+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-07-28T19:44:10.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[82644]: audit 2024-07-28T19:44:09.180141+0000 mon.a (mon.0) 1112 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi100-68802-28", "overlaypool":"test-librbd-smithi100-68802-30"}]: dispatch 2024-07-28T19:44:10.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[82644]: audit 2024-07-28T19:44:10.148114+0000 mon.a (mon.0) 1113 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi100-68802-28", "overlaypool":"test-librbd-smithi100-68802-30"}]': finished 2024-07-28T19:44:10.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:10 smithi100 bash[82644]: cluster 2024-07-28T19:44:10.148262+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-07-28T19:44:10.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:10 smithi138 bash[67894]: cluster 2024-07-28T19:44:09.169459+0000 mon.a (mon.0) 1109 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-07-28T19:44:10.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:10 smithi138 bash[67894]: audit 2024-07-28T19:44:09.177919+0000 mon.a (mon.0) 1110 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi100-68802-30", "mode":"writeback"}]': finished 2024-07-28T19:44:10.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:10 smithi138 bash[67894]: cluster 2024-07-28T19:44:09.178072+0000 mon.a (mon.0) 1111 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-07-28T19:44:10.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:10 smithi138 bash[67894]: audit 2024-07-28T19:44:09.180141+0000 mon.a (mon.0) 1112 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi100-68802-28", "overlaypool":"test-librbd-smithi100-68802-30"}]: dispatch 2024-07-28T19:44:10.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:10 smithi138 bash[67894]: audit 2024-07-28T19:44:10.148114+0000 mon.a (mon.0) 1113 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi100-68802-28", "overlaypool":"test-librbd-smithi100-68802-30"}]': finished 2024-07-28T19:44:10.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:10 smithi138 bash[67894]: cluster 2024-07-28T19:44:10.148262+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-07-28T19:44:11.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:11 smithi100 bash[80199]: cluster 2024-07-28T19:44:09.845242+0000 mgr.y (mgr.25358) 347 : cluster [DBG] pgmap v644: 617 pgs: 2 creating+peering, 7 unknown, 608 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:44:11.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:11 smithi100 bash[82644]: cluster 2024-07-28T19:44:09.845242+0000 mgr.y (mgr.25358) 347 : cluster [DBG] pgmap v644: 617 pgs: 2 creating+peering, 7 unknown, 608 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:44:11.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:11 smithi138 bash[67894]: cluster 2024-07-28T19:44:09.845242+0000 mgr.y (mgr.25358) 347 : cluster [DBG] pgmap v644: 617 pgs: 2 creating+peering, 7 unknown, 608 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:44:12.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:12 smithi100 bash[80199]: cluster 2024-07-28T19:44:11.212110+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-07-28T19:44:12.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:12 smithi100 bash[82644]: cluster 2024-07-28T19:44:11.212110+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-07-28T19:44:12.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:12 smithi138 bash[67894]: cluster 2024-07-28T19:44:11.212110+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-07-28T19:44:12.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:44:12 smithi138 bash[60615]: ts=2024-07-28T19:44:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:44:13.239 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:44:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:44:12] "GET /metrics HTTP/1.1" 200 75071 "" "Prometheus/2.43.0" 2024-07-28T19:44:13.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:13 smithi100 bash[80199]: cluster 2024-07-28T19:44:11.846348+0000 mgr.y (mgr.25358) 348 : cluster [DBG] pgmap v647: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:44:13.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:13 smithi100 bash[80199]: cluster 2024-07-28T19:44:12.232487+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-07-28T19:44:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:13 smithi100 bash[82644]: cluster 2024-07-28T19:44:11.846348+0000 mgr.y (mgr.25358) 348 : cluster [DBG] pgmap v647: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:44:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:13 smithi100 bash[82644]: cluster 2024-07-28T19:44:12.232487+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-07-28T19:44:13.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:13 smithi138 bash[67894]: cluster 2024-07-28T19:44:11.846348+0000 mgr.y (mgr.25358) 348 : cluster [DBG] pgmap v647: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:44:13.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:13 smithi138 bash[67894]: cluster 2024-07-28T19:44:12.232487+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-07-28T19:44:14.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:14 smithi138 bash[67894]: cluster 2024-07-28T19:44:13.236745+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-07-28T19:44:14.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:14 smithi100 bash[80199]: cluster 2024-07-28T19:44:13.236745+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-07-28T19:44:14.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:14 smithi100 bash[82644]: cluster 2024-07-28T19:44:13.236745+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-07-28T19:44:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:15 smithi138 bash[67894]: cluster 2024-07-28T19:44:13.849400+0000 mgr.y (mgr.25358) 349 : cluster [DBG] pgmap v650: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-07-28T19:44:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:15 smithi138 bash[67894]: cluster 2024-07-28T19:44:14.259684+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-07-28T19:44:15.557 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:44:15 smithi138 bash[59346]: logger=sqlstore.transactions t=2024-07-28T19:44:15.117352913Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-07-28T19:44:15.557 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:44:15 smithi138 bash[59346]: logger=cleanup t=2024-07-28T19:44:15.154040203Z level=info msg="Completed cleanup jobs" duration=48.131344ms 2024-07-28T19:44:15.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:15 smithi100 bash[80199]: cluster 2024-07-28T19:44:13.849400+0000 mgr.y (mgr.25358) 349 : cluster [DBG] pgmap v650: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-07-28T19:44:15.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:15 smithi100 bash[80199]: cluster 2024-07-28T19:44:14.259684+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-07-28T19:44:15.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:15 smithi100 bash[82644]: cluster 2024-07-28T19:44:13.849400+0000 mgr.y (mgr.25358) 349 : cluster [DBG] pgmap v650: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-07-28T19:44:15.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:15 smithi100 bash[82644]: cluster 2024-07-28T19:44:14.259684+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-07-28T19:44:16.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:16 smithi138 bash[67894]: cluster 2024-07-28T19:44:15.273309+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-07-28T19:44:16.582 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.582 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.582 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.582 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 1 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout:child: image101 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101/85fbcd2434c0 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.583 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 2 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:child: image102 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:child: image101 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101/85fbcd2434c0 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102/85fb2f54c6b6 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.584 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.585 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 3 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:child: image102 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:child: image101 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:child: image103 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101/85fbcd2434c0 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.586 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102/85fb2f54c6b6 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image103/85fb1d4c21a2 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.587 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 3 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:child: image102 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:child: image101 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:child: image103 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101/85fbcd2434c0 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102/85fb2f54c6b6 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image103/85fb1d4c21a2 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.588 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.589 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 4 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 4 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:child: image102 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:child: image101 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:child: image103 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:child: image104 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101/85fbcd2434c0 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102/85fb2f54c6b6 2024-07-28T19:44:16.590 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image103/85fb1d4c21a2 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image104/85fbf7286c1b 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.591 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:16.592 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:16.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:16 smithi100 bash[80199]: cluster 2024-07-28T19:44:15.273309+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-07-28T19:44:16.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:16 smithi100 bash[82644]: cluster 2024-07-28T19:44:15.273309+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-07-28T19:44:17.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:17 smithi138 bash[67894]: cluster 2024-07-28T19:44:15.850571+0000 mgr.y (mgr.25358) 350 : cluster [DBG] pgmap v653: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-07-28T19:44:17.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:17 smithi100 bash[80199]: cluster 2024-07-28T19:44:15.850571+0000 mgr.y (mgr.25358) 350 : cluster [DBG] pgmap v653: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-07-28T19:44:17.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:17 smithi100 bash[82644]: cluster 2024-07-28T19:44:15.850571+0000 mgr.y (mgr.25358) 350 : cluster [DBG] pgmap v653: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 2.7 KiB/s wr, 42 op/s 2024-07-28T19:44:19.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:19 smithi138 bash[67894]: cluster 2024-07-28T19:44:17.852408+0000 mgr.y (mgr.25358) 351 : cluster [DBG] pgmap v654: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 94 KiB/s rd, 3.2 KiB/s wr, 120 op/s 2024-07-28T19:44:19.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:19 smithi138 bash[67894]: audit 2024-07-28T19:44:18.260316+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:19.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:19 smithi138 bash[67894]: cluster 2024-07-28T19:44:18.291464+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-07-28T19:44:19.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:19 smithi138 bash[67894]: audit 2024-07-28T19:44:18.517365+0000 mon.a (mon.0) 1122 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi100-68802-28"}]: dispatch 2024-07-28T19:44:19.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:19 smithi100 bash[80199]: cluster 2024-07-28T19:44:17.852408+0000 mgr.y (mgr.25358) 351 : cluster [DBG] pgmap v654: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 94 KiB/s rd, 3.2 KiB/s wr, 120 op/s 2024-07-28T19:44:19.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:19 smithi100 bash[80199]: audit 2024-07-28T19:44:18.260316+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:19.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:19 smithi100 bash[80199]: cluster 2024-07-28T19:44:18.291464+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-07-28T19:44:19.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:19 smithi100 bash[80199]: audit 2024-07-28T19:44:18.517365+0000 mon.a (mon.0) 1122 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi100-68802-28"}]: dispatch 2024-07-28T19:44:19.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:19 smithi100 bash[82644]: cluster 2024-07-28T19:44:17.852408+0000 mgr.y (mgr.25358) 351 : cluster [DBG] pgmap v654: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 94 KiB/s rd, 3.2 KiB/s wr, 120 op/s 2024-07-28T19:44:19.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:19 smithi100 bash[82644]: audit 2024-07-28T19:44:18.260316+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:19.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:19 smithi100 bash[82644]: cluster 2024-07-28T19:44:18.291464+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-07-28T19:44:19.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:19 smithi100 bash[82644]: audit 2024-07-28T19:44:18.517365+0000 mon.a (mon.0) 1122 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi100-68802-28"}]: dispatch 2024-07-28T19:44:20.150 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image101 2024-07-28T19:44:20.150 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:20.150 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 4 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 4 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:child: image102 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:child: image101 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:child: image103 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:child: image104 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image101/85fbcd2434c0 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.151 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102/85fb2f54c6b6 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image103/85fb1d4c21a2 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image104/85fbf7286c1b 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.152 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image103 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.153 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 3 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout:child: image102 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout:child: image103 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout:child: image104 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102/85fb2f54c6b6 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image103/85fb1d4c21a2 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image104/85fbf7286c1b 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.154 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image104 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 2 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 2 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:child: image102 2024-07-28T19:44:20.155 INFO:tasks.workunit.client.0.smithi100.stdout:child: image104 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102/85fb2f54c6b6 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-29/image104/85fbf7286c1b 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:trying to find test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:checking test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 1 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 1 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:child: image102 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-28/image102/85fb2f54c6b6 2024-07-28T19:44:20.156 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:44:20.157 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 0 2024-07-28T19:44:20.157 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:44:20.157 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ListChildrenTiered (18731 ms) 2024-07-28T19:44:20.157 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.LockingPP 2024-07-28T19:44:20.157 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:20.305 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.LockingPP (155 ms) 2024-07-28T19:44:20.305 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.FlushAio 2024-07-28T19:44:20.306 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:20.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:20 smithi138 bash[67894]: audit 2024-07-28T19:44:19.295113+0000 mon.a (mon.0) 1123 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi100-68802-28"}]': finished 2024-07-28T19:44:20.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:20 smithi138 bash[67894]: cluster 2024-07-28T19:44:19.295254+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-07-28T19:44:20.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:20 smithi138 bash[67894]: audit 2024-07-28T19:44:19.299946+0000 mon.a (mon.0) 1125 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30"}]: dispatch 2024-07-28T19:44:20.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:20 smithi138 bash[67894]: cluster 2024-07-28T19:44:20.141978+0000 mon.a (mon.0) 1126 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-07-28T19:44:20.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:20 smithi138 bash[67894]: audit 2024-07-28T19:44:20.151441+0000 mon.a (mon.0) 1127 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30"}]': finished 2024-07-28T19:44:20.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:20 smithi138 bash[67894]: cluster 2024-07-28T19:44:20.151594+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-07-28T19:44:20.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[80199]: audit 2024-07-28T19:44:19.295113+0000 mon.a (mon.0) 1123 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi100-68802-28"}]': finished 2024-07-28T19:44:20.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[80199]: cluster 2024-07-28T19:44:19.295254+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-07-28T19:44:20.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[80199]: audit 2024-07-28T19:44:19.299946+0000 mon.a (mon.0) 1125 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30"}]: dispatch 2024-07-28T19:44:20.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[80199]: cluster 2024-07-28T19:44:20.141978+0000 mon.a (mon.0) 1126 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-07-28T19:44:20.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[80199]: audit 2024-07-28T19:44:20.151441+0000 mon.a (mon.0) 1127 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30"}]': finished 2024-07-28T19:44:20.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[80199]: cluster 2024-07-28T19:44:20.151594+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-07-28T19:44:20.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[82644]: audit 2024-07-28T19:44:19.295113+0000 mon.a (mon.0) 1123 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi100-68802-28"}]': finished 2024-07-28T19:44:20.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[82644]: cluster 2024-07-28T19:44:19.295254+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-07-28T19:44:20.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[82644]: audit 2024-07-28T19:44:19.299946+0000 mon.a (mon.0) 1125 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30"}]: dispatch 2024-07-28T19:44:20.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[82644]: cluster 2024-07-28T19:44:20.141978+0000 mon.a (mon.0) 1126 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-07-28T19:44:20.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[82644]: audit 2024-07-28T19:44:20.151441+0000 mon.a (mon.0) 1127 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi100-68802-28", "tierpool":"test-librbd-smithi100-68802-30"}]': finished 2024-07-28T19:44:20.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:20 smithi100 bash[82644]: cluster 2024-07-28T19:44:20.151594+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-07-28T19:44:20.754 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.FlushAio (449 ms) 2024-07-28T19:44:20.754 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-07-28T19:44:20.754 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:21.064 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.FlushAioPP (310 ms) 2024-07-28T19:44:21.064 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapDiff 2024-07-28T19:44:21.064 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:21.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:21 smithi100 bash[80199]: cluster 2024-07-28T19:44:19.854169+0000 mgr.y (mgr.25358) 352 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 347 KiB/s rd, 9.3 KiB/s wr, 426 op/s 2024-07-28T19:44:21.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:21 smithi100 bash[82644]: cluster 2024-07-28T19:44:19.854169+0000 mgr.y (mgr.25358) 352 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 347 KiB/s rd, 9.3 KiB/s wr, 426 op/s 2024-07-28T19:44:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:21 smithi138 bash[67894]: cluster 2024-07-28T19:44:19.854169+0000 mgr.y (mgr.25358) 352 : cluster [DBG] pgmap v657: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 347 KiB/s rd, 9.3 KiB/s wr, 426 op/s 2024-07-28T19:44:22.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:22 smithi100 bash[80199]: cluster 2024-07-28T19:44:21.312484+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-07-28T19:44:22.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:22 smithi100 bash[82644]: cluster 2024-07-28T19:44:21.312484+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-07-28T19:44:22.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:22 smithi138 bash[67894]: cluster 2024-07-28T19:44:21.312484+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-07-28T19:44:22.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:44:22 smithi138 bash[60615]: ts=2024-07-28T19:44:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:44:23.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:44:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:44:22] "GET /metrics HTTP/1.1" 200 75163 "" "Prometheus/2.43.0" 2024-07-28T19:44:23.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:23 smithi100 bash[80199]: cluster 2024-07-28T19:44:21.855269+0000 mgr.y (mgr.25358) 353 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 475 KiB/s rd, 12 KiB/s wr, 575 op/s 2024-07-28T19:44:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:23 smithi100 bash[80199]: cluster 2024-07-28T19:44:22.344206+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-07-28T19:44:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:23 smithi100 bash[80199]: audit 2024-07-28T19:44:22.638169+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:23 smithi100 bash[82644]: cluster 2024-07-28T19:44:21.855269+0000 mgr.y (mgr.25358) 353 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 475 KiB/s rd, 12 KiB/s wr, 575 op/s 2024-07-28T19:44:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:23 smithi100 bash[82644]: cluster 2024-07-28T19:44:22.344206+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-07-28T19:44:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:23 smithi100 bash[82644]: audit 2024-07-28T19:44:22.638169+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:23.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:23 smithi138 bash[67894]: cluster 2024-07-28T19:44:21.855269+0000 mgr.y (mgr.25358) 353 : cluster [DBG] pgmap v660: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 475 KiB/s rd, 12 KiB/s wr, 575 op/s 2024-07-28T19:44:23.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:23 smithi138 bash[67894]: cluster 2024-07-28T19:44:22.344206+0000 mon.a (mon.0) 1130 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-07-28T19:44:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:23 smithi138 bash[67894]: audit 2024-07-28T19:44:22.638169+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:24.610 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:44:24.611 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapDiff (3546 ms) 2024-07-28T19:44:24.611 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-07-28T19:44:24.611 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:24.671 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (61 ms) 2024-07-28T19:44:24.671 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-07-28T19:44:24.671 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:24.741 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (70 ms) 2024-07-28T19:44:24.741 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-07-28T19:44:24.742 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:24.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:24 smithi100 bash[80199]: cluster 2024-07-28T19:44:23.341575+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-07-28T19:44:24.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:24 smithi100 bash[82644]: cluster 2024-07-28T19:44:23.341575+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-07-28T19:44:24.786 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ZeroLengthRead (45 ms) 2024-07-28T19:44:24.787 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-07-28T19:44:24.787 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:24.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:24 smithi138 bash[67894]: cluster 2024-07-28T19:44:23.341575+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-07-28T19:44:24.866 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.LargeCacheRead (80 ms) 2024-07-28T19:44:24.866 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-07-28T19:44:24.866 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:25.289 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestPendingAio (423 ms) 2024-07-28T19:44:25.289 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-07-28T19:44:25.289 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:25.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:25 smithi100 bash[80199]: cluster 2024-07-28T19:44:23.858291+0000 mgr.y (mgr.25358) 354 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 9.2 KiB/s wr, 265 op/s 2024-07-28T19:44:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:25 smithi100 bash[80199]: cluster 2024-07-28T19:44:24.366212+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-07-28T19:44:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:25 smithi100 bash[80199]: cluster 2024-07-28T19:44:25.153854+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-07-28T19:44:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:25 smithi100 bash[82644]: cluster 2024-07-28T19:44:23.858291+0000 mgr.y (mgr.25358) 354 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 9.2 KiB/s wr, 265 op/s 2024-07-28T19:44:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:25 smithi100 bash[82644]: cluster 2024-07-28T19:44:24.366212+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-07-28T19:44:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:25 smithi100 bash[82644]: cluster 2024-07-28T19:44:25.153854+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-07-28T19:44:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:25 smithi138 bash[67894]: cluster 2024-07-28T19:44:23.858291+0000 mgr.y (mgr.25358) 354 : cluster [DBG] pgmap v663: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 9.2 KiB/s wr, 265 op/s 2024-07-28T19:44:25.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:25 smithi138 bash[67894]: cluster 2024-07-28T19:44:24.366212+0000 mon.a (mon.0) 1133 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-07-28T19:44:25.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:25 smithi138 bash[67894]: cluster 2024-07-28T19:44:25.153854+0000 mon.a (mon.0) 1134 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-07-28T19:44:26.382 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:27.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:27 smithi100 bash[80199]: cluster 2024-07-28T19:44:25.859571+0000 mgr.y (mgr.25358) 355 : cluster [DBG] pgmap v666: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 141 KiB/s wr, 294 op/s 2024-07-28T19:44:27.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:27 smithi100 bash[80199]: cluster 2024-07-28T19:44:26.158139+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-07-28T19:44:27.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:27 smithi100 bash[82644]: cluster 2024-07-28T19:44:25.859571+0000 mgr.y (mgr.25358) 355 : cluster [DBG] pgmap v666: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 141 KiB/s wr, 294 op/s 2024-07-28T19:44:27.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:27 smithi100 bash[82644]: cluster 2024-07-28T19:44:26.158139+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-07-28T19:44:27.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:27 smithi138 bash[67894]: cluster 2024-07-28T19:44:25.859571+0000 mgr.y (mgr.25358) 355 : cluster [DBG] pgmap v666: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 141 KiB/s wr, 294 op/s 2024-07-28T19:44:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:27 smithi138 bash[67894]: cluster 2024-07-28T19:44:26.158139+0000 mon.a (mon.0) 1135 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-07-28T19:44:28.486 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (3197 ms) 2024-07-28T19:44:28.487 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-07-28T19:44:28.487 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:28.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:28 smithi100 bash[80199]: cluster 2024-07-28T19:44:27.372204+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-07-28T19:44:28.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:28 smithi100 bash[82644]: cluster 2024-07-28T19:44:27.372204+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-07-28T19:44:28.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:28 smithi138 bash[67894]: cluster 2024-07-28T19:44:27.372204+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-07-28T19:44:29.611 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:29 smithi100 bash[80199]: cluster 2024-07-28T19:44:27.860696+0000 mgr.y (mgr.25358) 356 : cluster [DBG] pgmap v669: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 136 KiB/s wr, 195 op/s 2024-07-28T19:44:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:29 smithi100 bash[80199]: cluster 2024-07-28T19:44:28.387415+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-07-28T19:44:29.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:29 smithi100 bash[82644]: cluster 2024-07-28T19:44:27.860696+0000 mgr.y (mgr.25358) 356 : cluster [DBG] pgmap v669: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 136 KiB/s wr, 195 op/s 2024-07-28T19:44:29.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:29 smithi100 bash[82644]: cluster 2024-07-28T19:44:28.387415+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-07-28T19:44:29.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:29 smithi138 bash[67894]: cluster 2024-07-28T19:44:27.860696+0000 mgr.y (mgr.25358) 356 : cluster [DBG] pgmap v669: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 136 KiB/s wr, 195 op/s 2024-07-28T19:44:29.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:29 smithi138 bash[67894]: cluster 2024-07-28T19:44:28.387415+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-07-28T19:44:30.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:30 smithi100 bash[80199]: cluster 2024-07-28T19:44:29.391036+0000 mon.a (mon.0) 1138 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-07-28T19:44:30.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:30 smithi100 bash[80199]: cluster 2024-07-28T19:44:30.156264+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-07-28T19:44:30.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:30 smithi100 bash[82644]: cluster 2024-07-28T19:44:29.391036+0000 mon.a (mon.0) 1138 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-07-28T19:44:30.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:30 smithi100 bash[82644]: cluster 2024-07-28T19:44:30.156264+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-07-28T19:44:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:30 smithi138 bash[67894]: cluster 2024-07-28T19:44:29.391036+0000 mon.a (mon.0) 1138 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-07-28T19:44:30.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:30 smithi138 bash[67894]: cluster 2024-07-28T19:44:30.156264+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-07-28T19:44:31.269 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (2783 ms) 2024-07-28T19:44:31.269 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.Flatten 2024-07-28T19:44:31.270 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:31.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:31 smithi100 bash[80199]: cluster 2024-07-28T19:44:29.862466+0000 mgr.y (mgr.25358) 357 : cluster [DBG] pgmap v672: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 729 KiB/s rd, 594 KiB/s wr, 347 op/s 2024-07-28T19:44:31.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:31 smithi100 bash[80199]: cluster 2024-07-28T19:44:31.159182+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-07-28T19:44:31.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:31 smithi100 bash[82644]: cluster 2024-07-28T19:44:29.862466+0000 mgr.y (mgr.25358) 357 : cluster [DBG] pgmap v672: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 729 KiB/s rd, 594 KiB/s wr, 347 op/s 2024-07-28T19:44:31.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:31 smithi100 bash[82644]: cluster 2024-07-28T19:44:31.159182+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-07-28T19:44:31.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:31 smithi138 bash[67894]: cluster 2024-07-28T19:44:29.862466+0000 mgr.y (mgr.25358) 357 : cluster [DBG] pgmap v672: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 729 KiB/s rd, 594 KiB/s wr, 347 op/s 2024-07-28T19:44:31.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:31 smithi138 bash[67894]: cluster 2024-07-28T19:44:31.159182+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-07-28T19:44:32.392 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.Flatten (1123 ms) 2024-07-28T19:44:32.392 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.Sparsify 2024-07-28T19:44:32.392 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:32.480 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 32768 2024-07-28T19:44:32.480 INFO:tasks.workunit.client.0.smithi100.stdout:read: 32768 2024-07-28T19:44:32.480 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.Sparsify (88 ms) 2024-07-28T19:44:32.481 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-07-28T19:44:32.481 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:32.561 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SparsifyPP (81 ms) 2024-07-28T19:44:32.561 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-07-28T19:44:32.561 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:32.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:44:32 smithi138 bash[60615]: ts=2024-07-28T19:44:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:44:33.156 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:44:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:44:32] "GET /metrics HTTP/1.1" 200 75163 "" "Prometheus/2.43.0" 2024-07-28T19:44:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:33 smithi100 bash[80199]: cluster 2024-07-28T19:44:31.863660+0000 mgr.y (mgr.25358) 358 : cluster [DBG] pgmap v675: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 775 KiB/s rd, 600 KiB/s wr, 407 op/s 2024-07-28T19:44:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:33 smithi100 bash[80199]: cluster 2024-07-28T19:44:32.161939+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-07-28T19:44:33.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:33 smithi100 bash[82644]: cluster 2024-07-28T19:44:31.863660+0000 mgr.y (mgr.25358) 358 : cluster [DBG] pgmap v675: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 775 KiB/s rd, 600 KiB/s wr, 407 op/s 2024-07-28T19:44:33.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:33 smithi100 bash[82644]: cluster 2024-07-28T19:44:32.161939+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-07-28T19:44:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:33 smithi138 bash[67894]: cluster 2024-07-28T19:44:31.863660+0000 mgr.y (mgr.25358) 358 : cluster [DBG] pgmap v675: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 775 KiB/s rd, 600 KiB/s wr, 407 op/s 2024-07-28T19:44:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:33 smithi138 bash[67894]: cluster 2024-07-28T19:44:32.161939+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-07-28T19:44:34.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:34 smithi100 bash[80199]: cluster 2024-07-28T19:44:33.406145+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-07-28T19:44:34.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:34 smithi100 bash[82644]: cluster 2024-07-28T19:44:33.406145+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-07-28T19:44:34.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:34 smithi138 bash[67894]: cluster 2024-07-28T19:44:33.406145+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-07-28T19:44:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:35 smithi100 bash[80199]: cluster 2024-07-28T19:44:33.866753+0000 mgr.y (mgr.25358) 359 : cluster [DBG] pgmap v678: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 35 KiB/s wr, 157 op/s 2024-07-28T19:44:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:35 smithi100 bash[80199]: cluster 2024-07-28T19:44:34.417247+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-07-28T19:44:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:35 smithi100 bash[80199]: cluster 2024-07-28T19:44:35.158606+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-07-28T19:44:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:35 smithi100 bash[82644]: cluster 2024-07-28T19:44:33.866753+0000 mgr.y (mgr.25358) 359 : cluster [DBG] pgmap v678: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 35 KiB/s wr, 157 op/s 2024-07-28T19:44:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:35 smithi100 bash[82644]: cluster 2024-07-28T19:44:34.417247+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-07-28T19:44:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:35 smithi100 bash[82644]: cluster 2024-07-28T19:44:35.158606+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-07-28T19:44:35.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:35 smithi138 bash[67894]: cluster 2024-07-28T19:44:33.866753+0000 mgr.y (mgr.25358) 359 : cluster [DBG] pgmap v678: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 35 KiB/s wr, 157 op/s 2024-07-28T19:44:35.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:35 smithi138 bash[67894]: cluster 2024-07-28T19:44:34.417247+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-07-28T19:44:35.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:35 smithi138 bash[67894]: cluster 2024-07-28T19:44:35.158606+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-07-28T19:44:37.233 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapshotLimit (4672 ms) 2024-07-28T19:44:37.233 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-07-28T19:44:37.233 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:37 smithi100 bash[80199]: cluster 2024-07-28T19:44:35.867858+0000 mgr.y (mgr.25358) 360 : cluster [DBG] pgmap v681: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 221 KiB/s rd, 39 KiB/s wr, 272 op/s 2024-07-28T19:44:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:37 smithi100 bash[80199]: cluster 2024-07-28T19:44:36.162450+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-07-28T19:44:37.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:37 smithi100 bash[82644]: cluster 2024-07-28T19:44:35.867858+0000 mgr.y (mgr.25358) 360 : cluster [DBG] pgmap v681: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 221 KiB/s rd, 39 KiB/s wr, 272 op/s 2024-07-28T19:44:37.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:37 smithi100 bash[82644]: cluster 2024-07-28T19:44:36.162450+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-07-28T19:44:37.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:37 smithi138 bash[67894]: cluster 2024-07-28T19:44:35.867858+0000 mgr.y (mgr.25358) 360 : cluster [DBG] pgmap v681: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 221 KiB/s rd, 39 KiB/s wr, 272 op/s 2024-07-28T19:44:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:37 smithi138 bash[67894]: cluster 2024-07-28T19:44:36.162450+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-07-28T19:44:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:38 smithi100 bash[80199]: cluster 2024-07-28T19:44:37.177042+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-07-28T19:44:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:38 smithi100 bash[80199]: audit 2024-07-28T19:44:37.638234+0000 mon.a (mon.0) 1147 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:38 smithi100 bash[82644]: cluster 2024-07-28T19:44:37.177042+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-07-28T19:44:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:38 smithi100 bash[82644]: audit 2024-07-28T19:44:37.638234+0000 mon.a (mon.0) 1147 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:38.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:38 smithi138 bash[67894]: cluster 2024-07-28T19:44:37.177042+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-07-28T19:44:38.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:38 smithi138 bash[67894]: audit 2024-07-28T19:44:37.638234+0000 mon.a (mon.0) 1147 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[80199]: cluster 2024-07-28T19:44:37.869688+0000 mgr.y (mgr.25358) 361 : cluster [DBG] pgmap v684: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 14 KiB/s wr, 221 op/s 2024-07-28T19:44:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[80199]: cluster 2024-07-28T19:44:38.185504+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-07-28T19:44:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[80199]: audit 2024-07-28T19:44:38.223182+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]: dispatch 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[80199]: audit 2024-07-28T19:44:38.223963+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]: dispatch 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[80199]: audit 2024-07-28T19:44:38.224774+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[80199]: audit 2024-07-28T19:44:38.225392+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]: dispatch 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[80199]: audit 2024-07-28T19:44:38.225962+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.8", "id": [3, 5]}]: dispatch 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[82644]: cluster 2024-07-28T19:44:37.869688+0000 mgr.y (mgr.25358) 361 : cluster [DBG] pgmap v684: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 14 KiB/s wr, 221 op/s 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[82644]: cluster 2024-07-28T19:44:38.185504+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[82644]: audit 2024-07-28T19:44:38.223182+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]: dispatch 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[82644]: audit 2024-07-28T19:44:38.223963+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]: dispatch 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[82644]: audit 2024-07-28T19:44:38.224774+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-07-28T19:44:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[82644]: audit 2024-07-28T19:44:38.225392+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]: dispatch 2024-07-28T19:44:39.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:39 smithi100 bash[82644]: audit 2024-07-28T19:44:38.225962+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.8", "id": [3, 5]}]: dispatch 2024-07-28T19:44:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:39 smithi138 bash[67894]: cluster 2024-07-28T19:44:37.869688+0000 mgr.y (mgr.25358) 361 : cluster [DBG] pgmap v684: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 170 KiB/s rd, 14 KiB/s wr, 221 op/s 2024-07-28T19:44:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:39 smithi138 bash[67894]: cluster 2024-07-28T19:44:38.185504+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-07-28T19:44:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:39 smithi138 bash[67894]: audit 2024-07-28T19:44:38.223182+0000 mon.a (mon.0) 1149 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]: dispatch 2024-07-28T19:44:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:39 smithi138 bash[67894]: audit 2024-07-28T19:44:38.223963+0000 mon.a (mon.0) 1150 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]: dispatch 2024-07-28T19:44:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:39 smithi138 bash[67894]: audit 2024-07-28T19:44:38.224774+0000 mon.a (mon.0) 1151 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-07-28T19:44:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:39 smithi138 bash[67894]: audit 2024-07-28T19:44:38.225392+0000 mon.a (mon.0) 1152 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]: dispatch 2024-07-28T19:44:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:39 smithi138 bash[67894]: audit 2024-07-28T19:44:38.225962+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.8", "id": [3, 5]}]: dispatch 2024-07-28T19:44:40.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[80199]: audit 2024-07-28T19:44:39.195662+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-07-28T19:44:40.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[80199]: audit 2024-07-28T19:44:39.195843+0000 mon.a (mon.0) 1155 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-07-28T19:44:40.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[80199]: audit 2024-07-28T19:44:39.195940+0000 mon.a (mon.0) 1156 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-07-28T19:44:40.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[80199]: audit 2024-07-28T19:44:39.196145+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-07-28T19:44:40.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[80199]: audit 2024-07-28T19:44:39.196330+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.8", "id": [3, 5]}]': finished 2024-07-28T19:44:40.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[80199]: cluster 2024-07-28T19:44:39.196486+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-07-28T19:44:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[82644]: audit 2024-07-28T19:44:39.195662+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-07-28T19:44:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[82644]: audit 2024-07-28T19:44:39.195843+0000 mon.a (mon.0) 1155 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-07-28T19:44:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[82644]: audit 2024-07-28T19:44:39.195940+0000 mon.a (mon.0) 1156 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-07-28T19:44:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[82644]: audit 2024-07-28T19:44:39.196145+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-07-28T19:44:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[82644]: audit 2024-07-28T19:44:39.196330+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.8", "id": [3, 5]}]': finished 2024-07-28T19:44:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:40 smithi100 bash[82644]: cluster 2024-07-28T19:44:39.196486+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-07-28T19:44:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:40 smithi138 bash[67894]: audit 2024-07-28T19:44:39.195662+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-07-28T19:44:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:40 smithi138 bash[67894]: audit 2024-07-28T19:44:39.195843+0000 mon.a (mon.0) 1155 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.0", "id": [2, 5]}]': finished 2024-07-28T19:44:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:40 smithi138 bash[67894]: audit 2024-07-28T19:44:39.195940+0000 mon.a (mon.0) 1156 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-07-28T19:44:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:40 smithi138 bash[67894]: audit 2024-07-28T19:44:39.196145+0000 mon.a (mon.0) 1157 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-07-28T19:44:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:40 smithi138 bash[67894]: audit 2024-07-28T19:44:39.196330+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.8", "id": [3, 5]}]': finished 2024-07-28T19:44:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:40 smithi138 bash[67894]: cluster 2024-07-28T19:44:39.196486+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-07-28T19:44:41.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:41 smithi100 bash[80199]: cluster 2024-07-28T19:44:39.871335+0000 mgr.y (mgr.25358) 362 : cluster [DBG] pgmap v687: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 KiB/s wr, 37 op/s 2024-07-28T19:44:41.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:41 smithi100 bash[80199]: cluster 2024-07-28T19:44:40.219236+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-07-28T19:44:41.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:41 smithi100 bash[82644]: cluster 2024-07-28T19:44:39.871335+0000 mgr.y (mgr.25358) 362 : cluster [DBG] pgmap v687: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 KiB/s wr, 37 op/s 2024-07-28T19:44:41.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:41 smithi100 bash[82644]: cluster 2024-07-28T19:44:40.219236+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-07-28T19:44:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:41 smithi138 bash[67894]: cluster 2024-07-28T19:44:39.871335+0000 mgr.y (mgr.25358) 362 : cluster [DBG] pgmap v687: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 KiB/s wr, 37 op/s 2024-07-28T19:44:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:41 smithi138 bash[67894]: cluster 2024-07-28T19:44:40.219236+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-07-28T19:44:42.288 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5055 ms) 2024-07-28T19:44:42.288 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-07-28T19:44:42.288 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:42.430 INFO:tasks.workunit.client.0.smithi100.stdout:0.00% done 2024-07-28T19:44:42.430 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (142 ms) 2024-07-28T19:44:42.431 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-07-28T19:44:42.431 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:42.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:42 smithi100 bash[80199]: cluster 2024-07-28T19:44:41.209867+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-07-28T19:44:42.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:42 smithi100 bash[82644]: cluster 2024-07-28T19:44:41.209867+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-07-28T19:44:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:42 smithi138 bash[67894]: cluster 2024-07-28T19:44:41.209867+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-07-28T19:44:42.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:44:42 smithi138 bash[60615]: ts=2024-07-28T19:44:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:44:42.603 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (173 ms) 2024-07-28T19:44:42.604 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-07-28T19:44:42.604 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:43.223 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:44:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:44:42] "GET /metrics HTTP/1.1" 200 75161 "" "Prometheus/2.43.0" 2024-07-28T19:44:43.330 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (726 ms) 2024-07-28T19:44:43.330 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-07-28T19:44:43.330 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:43.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:43 smithi100 bash[80199]: cluster 2024-07-28T19:44:41.872470+0000 mgr.y (mgr.25358) 363 : cluster [DBG] pgmap v690: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 KiB/s wr, 18 op/s 2024-07-28T19:44:43.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:43 smithi100 bash[80199]: cluster 2024-07-28T19:44:42.231887+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-07-28T19:44:43.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:43 smithi100 bash[82644]: cluster 2024-07-28T19:44:41.872470+0000 mgr.y (mgr.25358) 363 : cluster [DBG] pgmap v690: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 KiB/s wr, 18 op/s 2024-07-28T19:44:43.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:43 smithi100 bash[82644]: cluster 2024-07-28T19:44:42.231887+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-07-28T19:44:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:43 smithi138 bash[67894]: cluster 2024-07-28T19:44:41.872470+0000 mgr.y (mgr.25358) 363 : cluster [DBG] pgmap v690: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 2.2 KiB/s wr, 18 op/s 2024-07-28T19:44:43.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:43 smithi138 bash[67894]: cluster 2024-07-28T19:44:42.231887+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-07-28T19:44:44.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:44 smithi100 bash[80199]: cluster 2024-07-28T19:44:43.242338+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-07-28T19:44:44.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:44 smithi100 bash[82644]: cluster 2024-07-28T19:44:43.242338+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-07-28T19:44:44.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:44 smithi138 bash[67894]: cluster 2024-07-28T19:44:43.242338+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-07-28T19:44:45.269 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (1940 ms) 2024-07-28T19:44:45.269 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-07-28T19:44:45.269 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:45.436 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (167 ms) 2024-07-28T19:44:45.436 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-07-28T19:44:45.436 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:44:45.436 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-07-28T19:44:45.437 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-07-28T19:44:45.437 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:45.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:45 smithi100 bash[80199]: cluster 2024-07-28T19:44:43.875445+0000 mgr.y (mgr.25358) 364 : cluster [DBG] pgmap v693: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.0 KiB/s wr, 35 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:45.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:45 smithi100 bash[80199]: cluster 2024-07-28T19:44:44.254960+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-07-28T19:44:45.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:45 smithi100 bash[80199]: cluster 2024-07-28T19:44:45.167683+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-07-28T19:44:45.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:45 smithi100 bash[82644]: cluster 2024-07-28T19:44:43.875445+0000 mgr.y (mgr.25358) 364 : cluster [DBG] pgmap v693: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.0 KiB/s wr, 35 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:45.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:45 smithi100 bash[82644]: cluster 2024-07-28T19:44:44.254960+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-07-28T19:44:45.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:45 smithi100 bash[82644]: cluster 2024-07-28T19:44:45.167683+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-07-28T19:44:45.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:45 smithi138 bash[67894]: cluster 2024-07-28T19:44:43.875445+0000 mgr.y (mgr.25358) 364 : cluster [DBG] pgmap v693: 617 pgs: 3 peering, 614 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.0 KiB/s wr, 35 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:45.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:45 smithi138 bash[67894]: cluster 2024-07-28T19:44:44.254960+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-07-28T19:44:45.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:45 smithi138 bash[67894]: cluster 2024-07-28T19:44:45.167683+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-07-28T19:44:47.334 INFO:tasks.workunit.client.0.smithi100.stdout:100.00% done 2024-07-28T19:44:47.334 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapRemove2 (1898 ms) 2024-07-28T19:44:47.334 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-07-28T19:44:47.334 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:47.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:47 smithi138 bash[67894]: cluster 2024-07-28T19:44:45.876546+0000 mgr.y (mgr.25358) 365 : cluster [DBG] pgmap v696: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 11 KiB/s wr, 172 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:47.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:47 smithi138 bash[67894]: cluster 2024-07-28T19:44:46.260212+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-07-28T19:44:47.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:47 smithi100 bash[80199]: cluster 2024-07-28T19:44:45.876546+0000 mgr.y (mgr.25358) 365 : cluster [DBG] pgmap v696: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 11 KiB/s wr, 172 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:47.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:47 smithi100 bash[80199]: cluster 2024-07-28T19:44:46.260212+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-07-28T19:44:47.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:47 smithi100 bash[82644]: cluster 2024-07-28T19:44:45.876546+0000 mgr.y (mgr.25358) 365 : cluster [DBG] pgmap v696: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 11 KiB/s wr, 172 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:47.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:47 smithi100 bash[82644]: cluster 2024-07-28T19:44:46.260212+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-07-28T19:44:48.382 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1048 ms) 2024-07-28T19:44:48.382 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-07-28T19:44:48.382 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:48.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:48 smithi138 bash[67894]: cluster 2024-07-28T19:44:47.263758+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-07-28T19:44:48.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:48 smithi100 bash[80199]: cluster 2024-07-28T19:44:47.263758+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-07-28T19:44:48.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:48 smithi100 bash[82644]: cluster 2024-07-28T19:44:47.263758+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-07-28T19:44:49.387 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (1004 ms) 2024-07-28T19:44:49.387 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-07-28T19:44:49.387 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:49.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:49 smithi138 bash[67894]: cluster 2024-07-28T19:44:47.878456+0000 mgr.y (mgr.25358) 366 : cluster [DBG] pgmap v699: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 9.7 KiB/s wr, 158 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:49.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:49 smithi138 bash[67894]: cluster 2024-07-28T19:44:48.286932+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-07-28T19:44:49.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:49 smithi100 bash[80199]: cluster 2024-07-28T19:44:47.878456+0000 mgr.y (mgr.25358) 366 : cluster [DBG] pgmap v699: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 9.7 KiB/s wr, 158 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:49.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:49 smithi100 bash[80199]: cluster 2024-07-28T19:44:48.286932+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-07-28T19:44:49.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:49 smithi100 bash[82644]: cluster 2024-07-28T19:44:47.878456+0000 mgr.y (mgr.25358) 366 : cluster [DBG] pgmap v699: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 9.7 KiB/s wr, 158 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:44:49.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:49 smithi100 bash[82644]: cluster 2024-07-28T19:44:48.286932+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-07-28T19:44:50.314 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (926 ms) 2024-07-28T19:44:50.314 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-07-28T19:44:50.314 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:50 smithi138 bash[67894]: cluster 2024-07-28T19:44:49.291150+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-07-28T19:44:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:50 smithi138 bash[67894]: cluster 2024-07-28T19:44:50.169542+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-07-28T19:44:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:50 smithi100 bash[80199]: cluster 2024-07-28T19:44:49.291150+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-07-28T19:44:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:50 smithi100 bash[80199]: cluster 2024-07-28T19:44:50.169542+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-07-28T19:44:50.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:50 smithi100 bash[82644]: cluster 2024-07-28T19:44:49.291150+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-07-28T19:44:50.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:50 smithi100 bash[82644]: cluster 2024-07-28T19:44:50.169542+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-07-28T19:44:51.569 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1255 ms) 2024-07-28T19:44:51.569 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-07-28T19:44:51.569 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:51.674 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (106 ms) 2024-07-28T19:44:51.674 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-07-28T19:44:51.674 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:51 smithi100 bash[80199]: cluster 2024-07-28T19:44:49.880211+0000 mgr.y (mgr.25358) 367 : cluster [DBG] pgmap v702: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 11 KiB/s wr, 170 op/s 2024-07-28T19:44:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:51 smithi100 bash[82644]: cluster 2024-07-28T19:44:49.880211+0000 mgr.y (mgr.25358) 367 : cluster [DBG] pgmap v702: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 11 KiB/s wr, 170 op/s 2024-07-28T19:44:51.766 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (92 ms) 2024-07-28T19:44:51.766 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-07-28T19:44:51.766 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:44:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:51 smithi138 bash[67894]: cluster 2024-07-28T19:44:49.880211+0000 mgr.y (mgr.25358) 367 : cluster [DBG] pgmap v702: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 11 KiB/s wr, 170 op/s 2024-07-28T19:44:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:52 smithi100 bash[80199]: cluster 2024-07-28T19:44:51.328361+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-07-28T19:44:52.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:52 smithi100 bash[82644]: cluster 2024-07-28T19:44:51.328361+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-07-28T19:44:52.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:52 smithi138 bash[67894]: cluster 2024-07-28T19:44:51.328361+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-07-28T19:44:52.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:44:52 smithi138 bash[60615]: ts=2024-07-28T19:44:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:44:53.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:44:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:44:52] "GET /metrics HTTP/1.1" 200 75159 "" "Prometheus/2.43.0" 2024-07-28T19:44:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:53 smithi100 bash[80199]: cluster 2024-07-28T19:44:51.881374+0000 mgr.y (mgr.25358) 368 : cluster [DBG] pgmap v705: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 9.2 KiB/s wr, 151 op/s 2024-07-28T19:44:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:53 smithi100 bash[80199]: cluster 2024-07-28T19:44:52.345899+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-07-28T19:44:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:53 smithi100 bash[80199]: audit 2024-07-28T19:44:52.638322+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:53 smithi100 bash[82644]: cluster 2024-07-28T19:44:51.881374+0000 mgr.y (mgr.25358) 368 : cluster [DBG] pgmap v705: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 9.2 KiB/s wr, 151 op/s 2024-07-28T19:44:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:53 smithi100 bash[82644]: cluster 2024-07-28T19:44:52.345899+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-07-28T19:44:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:53 smithi100 bash[82644]: audit 2024-07-28T19:44:52.638322+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:53 smithi138 bash[67894]: cluster 2024-07-28T19:44:51.881374+0000 mgr.y (mgr.25358) 368 : cluster [DBG] pgmap v705: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 9.2 KiB/s wr, 151 op/s 2024-07-28T19:44:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:53 smithi138 bash[67894]: cluster 2024-07-28T19:44:52.345899+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-07-28T19:44:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:53 smithi138 bash[67894]: audit 2024-07-28T19:44:52.638322+0000 mon.a (mon.0) 1173 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:44:54.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:54 smithi100 bash[80199]: cluster 2024-07-28T19:44:53.348631+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-07-28T19:44:54.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:54 smithi100 bash[80199]: audit 2024-07-28T19:44:53.371499+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:54.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:54 smithi100 bash[82644]: cluster 2024-07-28T19:44:53.348631+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-07-28T19:44:54.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:54 smithi100 bash[82644]: audit 2024-07-28T19:44:53.371499+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:54.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:54 smithi138 bash[67894]: cluster 2024-07-28T19:44:53.348631+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-07-28T19:44:54.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:54 smithi138 bash[67894]: audit 2024-07-28T19:44:53.371499+0000 mon.a (mon.0) 1175 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:44:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:55 smithi100 bash[80199]: cluster 2024-07-28T19:44:53.884312+0000 mgr.y (mgr.25358) 369 : cluster [DBG] pgmap v708: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 6.0 KiB/s wr, 155 op/s 2024-07-28T19:44:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:55 smithi100 bash[80199]: cluster 2024-07-28T19:44:54.380303+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-07-28T19:44:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:55 smithi100 bash[80199]: cluster 2024-07-28T19:44:55.172445+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-07-28T19:44:55.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:55 smithi100 bash[82644]: cluster 2024-07-28T19:44:53.884312+0000 mgr.y (mgr.25358) 369 : cluster [DBG] pgmap v708: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 6.0 KiB/s wr, 155 op/s 2024-07-28T19:44:55.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:55 smithi100 bash[82644]: cluster 2024-07-28T19:44:54.380303+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-07-28T19:44:55.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:55 smithi100 bash[82644]: cluster 2024-07-28T19:44:55.172445+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-07-28T19:44:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:55 smithi138 bash[67894]: cluster 2024-07-28T19:44:53.884312+0000 mgr.y (mgr.25358) 369 : cluster [DBG] pgmap v708: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 6.0 KiB/s wr, 155 op/s 2024-07-28T19:44:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:55 smithi138 bash[67894]: cluster 2024-07-28T19:44:54.380303+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-07-28T19:44:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:55 smithi138 bash[67894]: cluster 2024-07-28T19:44:55.172445+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-07-28T19:44:57.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:57 smithi100 bash[82644]: cluster 2024-07-28T19:44:55.885456+0000 mgr.y (mgr.25358) 370 : cluster [DBG] pgmap v711: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 232 KiB/s rd, 13 KiB/s wr, 306 op/s 2024-07-28T19:44:57.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:57 smithi100 bash[82644]: cluster 2024-07-28T19:44:56.176674+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-07-28T19:44:57.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:57 smithi100 bash[80199]: cluster 2024-07-28T19:44:55.885456+0000 mgr.y (mgr.25358) 370 : cluster [DBG] pgmap v711: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 232 KiB/s rd, 13 KiB/s wr, 306 op/s 2024-07-28T19:44:57.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:57 smithi100 bash[80199]: cluster 2024-07-28T19:44:56.176674+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-07-28T19:44:57.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:57 smithi138 bash[67894]: cluster 2024-07-28T19:44:55.885456+0000 mgr.y (mgr.25358) 370 : cluster [DBG] pgmap v711: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 232 KiB/s rd, 13 KiB/s wr, 306 op/s 2024-07-28T19:44:57.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:57 smithi138 bash[67894]: cluster 2024-07-28T19:44:56.176674+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-07-28T19:44:58.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:58 smithi100 bash[80199]: cluster 2024-07-28T19:44:57.178392+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-07-28T19:44:58.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:58 smithi100 bash[82644]: cluster 2024-07-28T19:44:57.178392+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-07-28T19:44:58.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:58 smithi138 bash[67894]: cluster 2024-07-28T19:44:57.178392+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-07-28T19:44:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:59 smithi100 bash[80199]: cluster 2024-07-28T19:44:57.887327+0000 mgr.y (mgr.25358) 371 : cluster [DBG] pgmap v714: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 8.0 KiB/s wr, 153 op/s 2024-07-28T19:44:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:44:59 smithi100 bash[80199]: cluster 2024-07-28T19:44:58.203845+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-07-28T19:44:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:59 smithi100 bash[82644]: cluster 2024-07-28T19:44:57.887327+0000 mgr.y (mgr.25358) 371 : cluster [DBG] pgmap v714: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 8.0 KiB/s wr, 153 op/s 2024-07-28T19:44:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:44:59 smithi100 bash[82644]: cluster 2024-07-28T19:44:58.203845+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-07-28T19:44:59.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:59 smithi138 bash[67894]: cluster 2024-07-28T19:44:57.887327+0000 mgr.y (mgr.25358) 371 : cluster [DBG] pgmap v714: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 8.0 KiB/s wr, 153 op/s 2024-07-28T19:44:59.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:44:59 smithi138 bash[67894]: cluster 2024-07-28T19:44:58.203845+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-07-28T19:45:00.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:00 smithi100 bash[80199]: cluster 2024-07-28T19:44:59.204968+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-07-28T19:45:00.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:00 smithi100 bash[82644]: cluster 2024-07-28T19:44:59.204968+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-07-28T19:45:00.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:00 smithi138 bash[67894]: cluster 2024-07-28T19:44:59.204968+0000 mon.a (mon.0) 1181 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-07-28T19:45:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:01 smithi100 bash[80199]: cluster 2024-07-28T19:44:59.889193+0000 mgr.y (mgr.25358) 372 : cluster [DBG] pgmap v717: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-07-28T19:45:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:01 smithi100 bash[80199]: cluster 2024-07-28T19:45:00.227972+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-07-28T19:45:01.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:01 smithi100 bash[82644]: cluster 2024-07-28T19:44:59.889193+0000 mgr.y (mgr.25358) 372 : cluster [DBG] pgmap v717: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-07-28T19:45:01.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:01 smithi100 bash[82644]: cluster 2024-07-28T19:45:00.227972+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-07-28T19:45:01.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:01 smithi138 bash[67894]: cluster 2024-07-28T19:44:59.889193+0000 mgr.y (mgr.25358) 372 : cluster [DBG] pgmap v717: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-07-28T19:45:01.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:01 smithi138 bash[67894]: cluster 2024-07-28T19:45:00.227972+0000 mon.a (mon.0) 1182 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-07-28T19:45:02.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:02 smithi100 bash[82644]: cluster 2024-07-28T19:45:01.227091+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-07-28T19:45:02.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:02 smithi100 bash[80199]: cluster 2024-07-28T19:45:01.227091+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-07-28T19:45:02.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:02 smithi138 bash[67894]: cluster 2024-07-28T19:45:01.227091+0000 mon.a (mon.0) 1183 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-07-28T19:45:02.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:45:02 smithi138 bash[60615]: ts=2024-07-28T19:45:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:45:02.663 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (10897 ms) 2024-07-28T19:45:02.664 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.Metadata 2024-07-28T19:45:02.664 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:03.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:45:02] "GET /metrics HTTP/1.1" 200 75159 "" "Prometheus/2.43.0" 2024-07-28T19:45:03.543 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.Metadata (880 ms) 2024-07-28T19:45:03.543 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-07-28T19:45:03.543 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:03 smithi138 bash[67894]: cluster 2024-07-28T19:45:01.889999+0000 mgr.y (mgr.25358) 373 : cluster [DBG] pgmap v720: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 3.5 KiB/s wr, 89 op/s 2024-07-28T19:45:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:03 smithi100 bash[80199]: cluster 2024-07-28T19:45:01.889999+0000 mgr.y (mgr.25358) 373 : cluster [DBG] pgmap v720: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 3.5 KiB/s wr, 89 op/s 2024-07-28T19:45:03.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:03 smithi100 bash[82644]: cluster 2024-07-28T19:45:01.889999+0000 mgr.y (mgr.25358) 373 : cluster [DBG] pgmap v720: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 3.5 KiB/s wr, 89 op/s 2024-07-28T19:45:04.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:04 smithi138 bash[67894]: cluster 2024-07-28T19:45:03.249233+0000 mon.a (mon.0) 1184 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-07-28T19:45:04.568 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.MetadataPP (1025 ms) 2024-07-28T19:45:04.568 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-07-28T19:45:04.568 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:04.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:04 smithi100 bash[82644]: cluster 2024-07-28T19:45:03.249233+0000 mon.a (mon.0) 1184 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-07-28T19:45:04.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:04 smithi100 bash[80199]: cluster 2024-07-28T19:45:03.249233+0000 mon.a (mon.0) 1184 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-07-28T19:45:04.824 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.UpdateFeatures (256 ms) 2024-07-28T19:45:04.825 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-07-28T19:45:04.825 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-07-28T19:45:04.825 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-07-28T19:45:04.825 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:05.430 INFO:tasks.workunit.client.0.smithi100.stdout:0.00% done 2024-07-28T19:45:05.430 INFO:tasks.workunit.client.0.smithi100.stdout:25.00% done 2024-07-28T19:45:05.430 INFO:tasks.workunit.client.0.smithi100.stdout:50.00% done 2024-07-28T19:45:05.430 INFO:tasks.workunit.client.0.smithi100.stdout:75.00% done 2024-07-28T19:45:05.431 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.RebuildObjectMap (606 ms) 2024-07-28T19:45:05.431 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-07-28T19:45:05.518 INFO:tasks.workunit.client.0.smithi100.stdout:0.00% done 2024-07-28T19:45:05.518 INFO:tasks.workunit.client.0.smithi100.stdout:25.00% done 2024-07-28T19:45:05.518 INFO:tasks.workunit.client.0.smithi100.stdout:50.00% done 2024-07-28T19:45:05.518 INFO:tasks.workunit.client.0.smithi100.stdout:75.00% done 2024-07-28T19:45:05.518 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (88 ms) 2024-07-28T19:45:05.518 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-07-28T19:45:05.518 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:05.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:05 smithi138 bash[67894]: cluster 2024-07-28T19:45:03.892945+0000 mgr.y (mgr.25358) 374 : cluster [DBG] pgmap v722: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 135 KiB/s rd, 36 KiB/s wr, 205 op/s 2024-07-28T19:45:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:05 smithi138 bash[67894]: cluster 2024-07-28T19:45:04.275406+0000 mon.a (mon.0) 1185 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-07-28T19:45:05.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:05 smithi100 bash[80199]: cluster 2024-07-28T19:45:03.892945+0000 mgr.y (mgr.25358) 374 : cluster [DBG] pgmap v722: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 135 KiB/s rd, 36 KiB/s wr, 205 op/s 2024-07-28T19:45:05.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:05 smithi100 bash[80199]: cluster 2024-07-28T19:45:04.275406+0000 mon.a (mon.0) 1185 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-07-28T19:45:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:05 smithi100 bash[82644]: cluster 2024-07-28T19:45:03.892945+0000 mgr.y (mgr.25358) 374 : cluster [DBG] pgmap v722: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 135 KiB/s rd, 36 KiB/s wr, 205 op/s 2024-07-28T19:45:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:05 smithi100 bash[82644]: cluster 2024-07-28T19:45:04.275406+0000 mon.a (mon.0) 1185 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-07-28T19:45:06.396 INFO:tasks.workunit.client.0.smithi100.stdout:0.00% done 2024-07-28T19:45:06.396 INFO:tasks.workunit.client.0.smithi100.stdout:25.00% done 2024-07-28T19:45:06.396 INFO:tasks.workunit.client.0.smithi100.stdout:50.00% done 2024-07-28T19:45:06.396 INFO:tasks.workunit.client.0.smithi100.stdout:75.00% done 2024-07-28T19:45:06.396 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.CheckObjectMap (878 ms) 2024-07-28T19:45:06.396 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-07-28T19:45:06.397 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:06.444 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.BlockingAIO (48 ms) 2024-07-28T19:45:06.444 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-07-28T19:45:06.444 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:06.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:06 smithi138 bash[67894]: cluster 2024-07-28T19:45:05.273502+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-07-28T19:45:06.641 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (197 ms) 2024-07-28T19:45:06.641 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-07-28T19:45:06.641 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:45:06.641 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-07-28T19:45:06.641 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-07-28T19:45:06.641 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:06.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:06 smithi100 bash[80199]: cluster 2024-07-28T19:45:05.273502+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-07-28T19:45:06.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:06 smithi100 bash[82644]: cluster 2024-07-28T19:45:05.273502+0000 mon.a (mon.0) 1186 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-07-28T19:45:07.474 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (832 ms) 2024-07-28T19:45:07.474 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-07-28T19:45:07.474 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:07.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:07 smithi138 bash[67894]: cluster 2024-07-28T19:45:05.894062+0000 mgr.y (mgr.25358) 375 : cluster [DBG] pgmap v725: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 174 KiB/s wr, 712 op/s 2024-07-28T19:45:07.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:07 smithi138 bash[67894]: audit 2024-07-28T19:45:06.288037+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:45:07.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:07 smithi138 bash[67894]: cluster 2024-07-28T19:45:06.302645+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-07-28T19:45:07.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:07 smithi100 bash[80199]: cluster 2024-07-28T19:45:05.894062+0000 mgr.y (mgr.25358) 375 : cluster [DBG] pgmap v725: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 174 KiB/s wr, 712 op/s 2024-07-28T19:45:07.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:07 smithi100 bash[80199]: audit 2024-07-28T19:45:06.288037+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:45:07.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:07 smithi100 bash[80199]: cluster 2024-07-28T19:45:06.302645+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-07-28T19:45:07.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:07 smithi100 bash[82644]: cluster 2024-07-28T19:45:05.894062+0000 mgr.y (mgr.25358) 375 : cluster [DBG] pgmap v725: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 174 KiB/s wr, 712 op/s 2024-07-28T19:45:07.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:07 smithi100 bash[82644]: audit 2024-07-28T19:45:06.288037+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:45:07.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:07 smithi100 bash[82644]: cluster 2024-07-28T19:45:06.302645+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-07-28T19:45:08.423 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (950 ms) 2024-07-28T19:45:08.423 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-07-28T19:45:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:08 smithi100 bash[80199]: cluster 2024-07-28T19:45:07.308215+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-07-28T19:45:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:08 smithi100 bash[80199]: audit 2024-07-28T19:45:07.639057+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:08.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:08 smithi100 bash[82644]: cluster 2024-07-28T19:45:07.308215+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-07-28T19:45:08.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:08 smithi100 bash[82644]: audit 2024-07-28T19:45:07.639057+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:08.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:08 smithi138 bash[67894]: cluster 2024-07-28T19:45:07.308215+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-07-28T19:45:08.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:08 smithi138 bash[67894]: audit 2024-07-28T19:45:07.639057+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:09.648 INFO:tasks.workunit.client.0.smithi100.stdout:0.00% done 2024-07-28T19:45:09.648 INFO:tasks.workunit.client.0.smithi100.stdout:100.00% done 2024-07-28T19:45:09.648 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestImageOptions (1225 ms) 2024-07-28T19:45:09.648 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-07-28T19:45:09.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:09 smithi100 bash[80199]: cluster 2024-07-28T19:45:07.895206+0000 mgr.y (mgr.25358) 376 : cluster [DBG] pgmap v728: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 465 KiB/s rd, 178 KiB/s wr, 732 op/s 2024-07-28T19:45:09.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:09 smithi100 bash[80199]: cluster 2024-07-28T19:45:08.319865+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-07-28T19:45:09.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:09 smithi100 bash[82644]: cluster 2024-07-28T19:45:07.895206+0000 mgr.y (mgr.25358) 376 : cluster [DBG] pgmap v728: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 465 KiB/s rd, 178 KiB/s wr, 732 op/s 2024-07-28T19:45:09.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:09 smithi100 bash[82644]: cluster 2024-07-28T19:45:08.319865+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-07-28T19:45:09.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:09 smithi138 bash[67894]: cluster 2024-07-28T19:45:07.895206+0000 mgr.y (mgr.25358) 376 : cluster [DBG] pgmap v728: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 465 KiB/s rd, 178 KiB/s wr, 732 op/s 2024-07-28T19:45:09.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:09 smithi138 bash[67894]: cluster 2024-07-28T19:45:08.319865+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-07-28T19:45:10.626 INFO:tasks.workunit.client.0.smithi100.stdout:0.00% done 2024-07-28T19:45:10.626 INFO:tasks.workunit.client.0.smithi100.stdout:100.00% done 2024-07-28T19:45:10.626 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (977 ms) 2024-07-28T19:45:10.626 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-07-28T19:45:10.626 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-07-28T19:45:10.626 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-07-28T19:45:10.626 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-07-28T19:45:10.626 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-07-28T19:45:10.627 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:wrote: 512 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:45:10.724 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:45:10.725 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:started write 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:write completion cb called! 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 0 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:finished write 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:45:10.726 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:45:10.727 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:45:10.728 INFO:tasks.workunit.client.0.smithi100.stdout:created completion 2024-07-28T19:45:10.728 INFO:tasks.workunit.client.0.smithi100.stdout:started read 2024-07-28T19:45:10.728 INFO:tasks.workunit.client.0.smithi100.stdout:read completion cb called! 2024-07-28T19:45:10.728 INFO:tasks.workunit.client.0.smithi100.stdout:return value is: 512 2024-07-28T19:45:10.728 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ImagePollIO (98 ms) 2024-07-28T19:45:10.728 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.Mirror 2024-07-28T19:45:10.730 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:10.744 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:10.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:10 smithi100 bash[80199]: cluster 2024-07-28T19:45:09.335442+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-07-28T19:45:10.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:10 smithi100 bash[82644]: cluster 2024-07-28T19:45:09.335442+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-07-28T19:45:10.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:10 smithi138 bash[67894]: cluster 2024-07-28T19:45:09.335442+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-07-28T19:45:11.530 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.Mirror (806 ms) 2024-07-28T19:45:11.530 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-07-28T19:45:11.548 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (19 ms) 2024-07-28T19:45:11.548 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-07-28T19:45:11.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:11 smithi100 bash[80199]: cluster 2024-07-28T19:45:09.897149+0000 mgr.y (mgr.25358) 377 : cluster [DBG] pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 404 KiB/s rd, 114 KiB/s wr, 532 op/s 2024-07-28T19:45:11.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:11 smithi100 bash[80199]: cluster 2024-07-28T19:45:10.348012+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-07-28T19:45:11.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:11 smithi100 bash[82644]: cluster 2024-07-28T19:45:09.897149+0000 mgr.y (mgr.25358) 377 : cluster [DBG] pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 404 KiB/s rd, 114 KiB/s wr, 532 op/s 2024-07-28T19:45:11.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:11 smithi100 bash[82644]: cluster 2024-07-28T19:45:10.348012+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-07-28T19:45:11.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:11 smithi138 bash[67894]: cluster 2024-07-28T19:45:09.897149+0000 mgr.y (mgr.25358) 377 : cluster [DBG] pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 404 KiB/s rd, 114 KiB/s wr, 532 op/s 2024-07-28T19:45:11.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:11 smithi138 bash[67894]: cluster 2024-07-28T19:45:10.348012+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-07-28T19:45:12.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: cluster 2024-07-28T19:45:11.345583+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-07-28T19:45:12.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.493554+0000 mon.a (mon.0) 1195 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]: dispatch 2024-07-28T19:45:12.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.497724+0000 mon.a (mon.0) 1196 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.500461+0000 mon.a (mon.0) 1197 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/266c4d69-84b1-4483-ac11-f9f4334dae1a"}]: dispatch 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.505993+0000 mon.a (mon.0) 1198 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/266c4d69-84b1-4483-ac11-f9f4334dae1a"}]': finished 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.513136+0000 mon.a (mon.0) 1199 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/333c1d2d-3980-4970-9989-f1b2c515ba85"}]: dispatch 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.516364+0000 mon.a (mon.0) 1200 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/333c1d2d-3980-4970-9989-f1b2c515ba85"}]': finished 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.518615+0000 mon.a (mon.0) 1201 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/23b811b5-c170-4b27-85b1-e28ec359f4b1"}]: dispatch 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.524311+0000 mon.a (mon.0) 1202 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/23b811b5-c170-4b27-85b1-e28ec359f4b1"}]': finished 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.536598+0000 mon.a (mon.0) 1203 : audit [DBG] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.540828+0000 mon.a (mon.0) 1204 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.541223+0000 mon.a (mon.0) 1205 : audit [DBG] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.541524+0000 mon.a (mon.0) 1206 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.547115+0000 mon.a (mon.0) 1207 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]': finished 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.954226+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:11.960521+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: cluster 2024-07-28T19:45:11.345583+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.493554+0000 mon.a (mon.0) 1195 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]: dispatch 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.497724+0000 mon.a (mon.0) 1196 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.500461+0000 mon.a (mon.0) 1197 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/266c4d69-84b1-4483-ac11-f9f4334dae1a"}]: dispatch 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.505993+0000 mon.a (mon.0) 1198 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/266c4d69-84b1-4483-ac11-f9f4334dae1a"}]': finished 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.513136+0000 mon.a (mon.0) 1199 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/333c1d2d-3980-4970-9989-f1b2c515ba85"}]: dispatch 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.516364+0000 mon.a (mon.0) 1200 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/333c1d2d-3980-4970-9989-f1b2c515ba85"}]': finished 2024-07-28T19:45:12.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.518615+0000 mon.a (mon.0) 1201 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/23b811b5-c170-4b27-85b1-e28ec359f4b1"}]: dispatch 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.524311+0000 mon.a (mon.0) 1202 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/23b811b5-c170-4b27-85b1-e28ec359f4b1"}]': finished 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.536598+0000 mon.a (mon.0) 1203 : audit [DBG] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.540828+0000 mon.a (mon.0) 1204 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.541223+0000 mon.a (mon.0) 1205 : audit [DBG] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.541524+0000 mon.a (mon.0) 1206 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.547115+0000 mon.a (mon.0) 1207 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]': finished 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.954226+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:11.960521+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:12.199496+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[82644]: audit 2024-07-28T19:45:12.205561+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:12.199496+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.752 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[80199]: audit 2024-07-28T19:45:12.205561+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[60615]: ts=2024-07-28T19:45:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: cluster 2024-07-28T19:45:11.345583+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.493554+0000 mon.a (mon.0) 1195 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]: dispatch 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.497724+0000 mon.a (mon.0) 1196 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.500461+0000 mon.a (mon.0) 1197 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/266c4d69-84b1-4483-ac11-f9f4334dae1a"}]: dispatch 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.505993+0000 mon.a (mon.0) 1198 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/266c4d69-84b1-4483-ac11-f9f4334dae1a"}]': finished 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.513136+0000 mon.a (mon.0) 1199 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/333c1d2d-3980-4970-9989-f1b2c515ba85"}]: dispatch 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.516364+0000 mon.a (mon.0) 1200 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/333c1d2d-3980-4970-9989-f1b2c515ba85"}]': finished 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.518615+0000 mon.a (mon.0) 1201 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/23b811b5-c170-4b27-85b1-e28ec359f4b1"}]: dispatch 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.524311+0000 mon.a (mon.0) 1202 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/23b811b5-c170-4b27-85b1-e28ec359f4b1"}]': finished 2024-07-28T19:45:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.536598+0000 mon.a (mon.0) 1203 : audit [DBG] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.540828+0000 mon.a (mon.0) 1204 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' 2024-07-28T19:45:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.541223+0000 mon.a (mon.0) 1205 : audit [DBG] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.541524+0000 mon.a (mon.0) 1206 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]: dispatch 2024-07-28T19:45:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.547115+0000 mon.a (mon.0) 1207 : audit [INF] from='client.? 172.21.15.100:0/183336803' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/aca6af09-442b-42c2-8e4f-ef19bd157466"}]': finished 2024-07-28T19:45:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.954226+0000 mon.a (mon.0) 1208 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:11.960521+0000 mon.a (mon.0) 1209 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:12.199496+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:12.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:12 smithi138 bash[67894]: audit 2024-07-28T19:45:12.205561+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:13.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:45:12] "GET /metrics HTTP/1.1" 200 75159 "" "Prometheus/2.43.0" 2024-07-28T19:45:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[80199]: cluster 2024-07-28T19:45:11.897853+0000 mgr.y (mgr.25358) 378 : cluster [DBG] pgmap v734: 617 pgs: 617 active+clean; 140 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 389 KiB/s rd, 368 KiB/s wr, 510 op/s 2024-07-28T19:45:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[80199]: cluster 2024-07-28T19:45:12.561788+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-07-28T19:45:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[80199]: audit 2024-07-28T19:45:12.620708+0000 mon.a (mon.0) 1213 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:45:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[80199]: audit 2024-07-28T19:45:12.622217+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:45:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[80199]: audit 2024-07-28T19:45:12.632828+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[82644]: cluster 2024-07-28T19:45:11.897853+0000 mgr.y (mgr.25358) 378 : cluster [DBG] pgmap v734: 617 pgs: 617 active+clean; 140 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 389 KiB/s rd, 368 KiB/s wr, 510 op/s 2024-07-28T19:45:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[82644]: cluster 2024-07-28T19:45:12.561788+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-07-28T19:45:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[82644]: audit 2024-07-28T19:45:12.620708+0000 mon.a (mon.0) 1213 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:45:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[82644]: audit 2024-07-28T19:45:12.622217+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:45:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:13 smithi100 bash[82644]: audit 2024-07-28T19:45:12.632828+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:13 smithi138 bash[67894]: cluster 2024-07-28T19:45:11.897853+0000 mgr.y (mgr.25358) 378 : cluster [DBG] pgmap v734: 617 pgs: 617 active+clean; 140 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 389 KiB/s rd, 368 KiB/s wr, 510 op/s 2024-07-28T19:45:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:13 smithi138 bash[67894]: cluster 2024-07-28T19:45:12.561788+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-07-28T19:45:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:13 smithi138 bash[67894]: audit 2024-07-28T19:45:12.620708+0000 mon.a (mon.0) 1213 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:45:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:13 smithi138 bash[67894]: audit 2024-07-28T19:45:12.622217+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:45:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:13 smithi138 bash[67894]: audit 2024-07-28T19:45:12.632828+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:14.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:14 smithi100 bash[80199]: cluster 2024-07-28T19:45:13.639049+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-07-28T19:45:14.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:14 smithi100 bash[80199]: cluster 2024-07-28T19:45:13.899515+0000 mgr.y (mgr.25358) 379 : cluster [DBG] pgmap v737: 617 pgs: 617 active+clean; 144 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 242 KiB/s rd, 1.3 MiB/s wr, 315 op/s 2024-07-28T19:45:14.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:14 smithi100 bash[82644]: cluster 2024-07-28T19:45:13.639049+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-07-28T19:45:14.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:14 smithi100 bash[82644]: cluster 2024-07-28T19:45:13.899515+0000 mgr.y (mgr.25358) 379 : cluster [DBG] pgmap v737: 617 pgs: 617 active+clean; 144 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 242 KiB/s rd, 1.3 MiB/s wr, 315 op/s 2024-07-28T19:45:15.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:14 smithi138 bash[67894]: cluster 2024-07-28T19:45:13.639049+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-07-28T19:45:15.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:14 smithi138 bash[67894]: cluster 2024-07-28T19:45:13.899515+0000 mgr.y (mgr.25358) 379 : cluster [DBG] pgmap v737: 617 pgs: 617 active+clean; 144 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 242 KiB/s rd, 1.3 MiB/s wr, 315 op/s 2024-07-28T19:45:15.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:15 smithi100 bash[80199]: cluster 2024-07-28T19:45:14.651243+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-07-28T19:45:15.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:15 smithi100 bash[82644]: cluster 2024-07-28T19:45:14.651243+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-07-28T19:45:16.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:15 smithi138 bash[67894]: cluster 2024-07-28T19:45:14.651243+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-07-28T19:45:16.757 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (5208 ms) 2024-07-28T19:45:16.757 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-07-28T19:45:16.757 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:16.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:16 smithi100 bash[80199]: cluster 2024-07-28T19:45:15.659566+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-07-28T19:45:16.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:16 smithi100 bash[80199]: cluster 2024-07-28T19:45:15.900627+0000 mgr.y (mgr.25358) 380 : cluster [DBG] pgmap v740: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 273 KiB/s rd, 1.8 MiB/s wr, 366 op/s 2024-07-28T19:45:16.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:16 smithi100 bash[82644]: cluster 2024-07-28T19:45:15.659566+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-07-28T19:45:16.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:16 smithi100 bash[82644]: cluster 2024-07-28T19:45:15.900627+0000 mgr.y (mgr.25358) 380 : cluster [DBG] pgmap v740: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 273 KiB/s rd, 1.8 MiB/s wr, 366 op/s 2024-07-28T19:45:17.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:16 smithi138 bash[67894]: cluster 2024-07-28T19:45:15.659566+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-07-28T19:45:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:16 smithi138 bash[67894]: cluster 2024-07-28T19:45:15.900627+0000 mgr.y (mgr.25358) 380 : cluster [DBG] pgmap v740: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 273 KiB/s rd, 1.8 MiB/s wr, 366 op/s 2024-07-28T19:45:17.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:17 smithi100 bash[80199]: cluster 2024-07-28T19:45:16.669572+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-07-28T19:45:17.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:17 smithi100 bash[82644]: cluster 2024-07-28T19:45:16.669572+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-07-28T19:45:18.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:17 smithi138 bash[67894]: cluster 2024-07-28T19:45:16.669572+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-07-28T19:45:18.758 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (2002 ms) 2024-07-28T19:45:18.758 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-07-28T19:45:18.758 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:18.881 INFO:tasks.workunit.client.0.smithi100.stdout:2: acquiring exclusive lock 2024-07-28T19:45:18.881 INFO:tasks.workunit.client.0.smithi100.stdout:1: acquiring exclusive lock 2024-07-28T19:45:18.887 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock acquired 2024-07-28T19:45:18.905 INFO:tasks.workunit.client.0.smithi100.stdout:2: releasing exclusive lock 2024-07-28T19:45:18.912 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock released 2024-07-28T19:45:18.912 INFO:tasks.workunit.client.0.smithi100.stdout:2: acquiring exclusive lock 2024-07-28T19:45:18.914 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock acquired 2024-07-28T19:45:18.960 INFO:tasks.workunit.client.0.smithi100.stdout:1: releasing exclusive lock 2024-07-28T19:45:18.966 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock released 2024-07-28T19:45:18.966 INFO:tasks.workunit.client.0.smithi100.stdout:1: acquiring exclusive lock 2024-07-28T19:45:18.975 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock acquired 2024-07-28T19:45:18.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:18 smithi100 bash[80199]: cluster 2024-07-28T19:45:17.671197+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-07-28T19:45:18.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:18 smithi100 bash[80199]: cluster 2024-07-28T19:45:17.901755+0000 mgr.y (mgr.25358) 381 : cluster [DBG] pgmap v743: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 711 KiB/s wr, 91 op/s 2024-07-28T19:45:18.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:18 smithi100 bash[82644]: cluster 2024-07-28T19:45:17.671197+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-07-28T19:45:18.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:18 smithi100 bash[82644]: cluster 2024-07-28T19:45:17.901755+0000 mgr.y (mgr.25358) 381 : cluster [DBG] pgmap v743: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 711 KiB/s wr, 91 op/s 2024-07-28T19:45:19.018 INFO:tasks.workunit.client.0.smithi100.stdout:1: releasing exclusive lock 2024-07-28T19:45:19.025 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock released 2024-07-28T19:45:19.025 INFO:tasks.workunit.client.0.smithi100.stdout:1: acquiring exclusive lock 2024-07-28T19:45:19.032 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock acquired 2024-07-28T19:45:19.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:18 smithi138 bash[67894]: cluster 2024-07-28T19:45:17.671197+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-07-28T19:45:19.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:18 smithi138 bash[67894]: cluster 2024-07-28T19:45:17.901755+0000 mgr.y (mgr.25358) 381 : cluster [DBG] pgmap v743: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 711 KiB/s wr, 91 op/s 2024-07-28T19:45:19.073 INFO:tasks.workunit.client.0.smithi100.stdout:2: releasing exclusive lock 2024-07-28T19:45:19.080 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock released 2024-07-28T19:45:19.080 INFO:tasks.workunit.client.0.smithi100.stdout:2: acquiring exclusive lock 2024-07-28T19:45:19.088 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock acquired 2024-07-28T19:45:19.096 INFO:tasks.workunit.client.0.smithi100.stdout:1: releasing exclusive lock 2024-07-28T19:45:19.103 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock released 2024-07-28T19:45:19.103 INFO:tasks.workunit.client.0.smithi100.stdout:1: acquiring exclusive lock 2024-07-28T19:45:19.111 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock acquired 2024-07-28T19:45:19.151 INFO:tasks.workunit.client.0.smithi100.stdout:2: releasing exclusive lock 2024-07-28T19:45:19.157 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock released 2024-07-28T19:45:19.158 INFO:tasks.workunit.client.0.smithi100.stdout:2: acquiring exclusive lock 2024-07-28T19:45:19.165 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock acquired 2024-07-28T19:45:19.192 INFO:tasks.workunit.client.0.smithi100.stdout:1: releasing exclusive lock 2024-07-28T19:45:19.198 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock released 2024-07-28T19:45:19.198 INFO:tasks.workunit.client.0.smithi100.stdout:1: acquiring exclusive lock 2024-07-28T19:45:19.210 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock acquired 2024-07-28T19:45:19.224 INFO:tasks.workunit.client.0.smithi100.stdout:2: releasing exclusive lock 2024-07-28T19:45:19.231 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock released 2024-07-28T19:45:19.231 INFO:tasks.workunit.client.0.smithi100.stdout:2: acquiring exclusive lock 2024-07-28T19:45:19.241 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock acquired 2024-07-28T19:45:19.244 INFO:tasks.workunit.client.0.smithi100.stdout:1: releasing exclusive lock 2024-07-28T19:45:19.251 INFO:tasks.workunit.client.0.smithi100.stdout:1: exclusive lock released 2024-07-28T19:45:19.262 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock acquired 2024-07-28T19:45:19.262 INFO:tasks.workunit.client.0.smithi100.stdout:2: releasing exclusive lock 2024-07-28T19:45:19.268 INFO:tasks.workunit.client.0.smithi100.stdout:2: exclusive lock released 2024-07-28T19:45:19.309 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ExclusiveLock (550 ms) 2024-07-28T19:45:19.309 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.BreakLock 2024-07-28T19:45:19.345 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:19.775 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.BreakLock (467 ms) 2024-07-28T19:45:19.776 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-07-28T19:45:19.776 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:19.818 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (42 ms) 2024-07-28T19:45:19.818 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-07-28T19:45:19.818 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.DefaultFeatures (1 ms) 2024-07-28T19:45:19.818 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-07-28T19:45:19.818 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:19.992 INFO:tasks.workunit.client.0.smithi100.stdout:0.00% done 2024-07-28T19:45:19.992 INFO:tasks.workunit.client.0.smithi100.stdout:1.56% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:3.12% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:4.69% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:6.25% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:7.81% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:9.38% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:10.94% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:12.50% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:14.06% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:15.62% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:17.19% done 2024-07-28T19:45:19.993 INFO:tasks.workunit.client.0.smithi100.stdout:18.75% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:20.31% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:21.88% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:23.44% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:25.00% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:26.56% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:28.12% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:29.69% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:31.25% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:32.81% done 2024-07-28T19:45:19.994 INFO:tasks.workunit.client.0.smithi100.stdout:34.38% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:35.94% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:37.50% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:39.06% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:40.62% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:42.19% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:43.75% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:45.31% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:46.88% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:48.44% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:50.00% done 2024-07-28T19:45:19.995 INFO:tasks.workunit.client.0.smithi100.stdout:51.56% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:53.12% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:54.69% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:56.25% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:57.81% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:59.38% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:60.94% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:62.50% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:64.06% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:65.62% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:67.19% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:68.75% done 2024-07-28T19:45:19.996 INFO:tasks.workunit.client.0.smithi100.stdout:70.31% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:71.88% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:73.44% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:75.00% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:76.56% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:78.12% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:79.69% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:81.25% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:82.81% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:84.38% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:85.94% done 2024-07-28T19:45:19.997 INFO:tasks.workunit.client.0.smithi100.stdout:87.50% done 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:89.06% done 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:90.62% done 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:92.19% done 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:93.75% done 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:95.31% done 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:96.88% done 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:98.44% done 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (174 ms) 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-07-28T19:45:19.998 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:19.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:19 smithi100 bash[80199]: cluster 2024-07-28T19:45:18.694631+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-07-28T19:45:19.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:19 smithi100 bash[80199]: audit 2024-07-28T19:45:19.437535+0000 mon.a (mon.0) 1222 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/1274339179"}]: dispatch 2024-07-28T19:45:19.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:19 smithi100 bash[82644]: cluster 2024-07-28T19:45:18.694631+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-07-28T19:45:19.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:19 smithi100 bash[82644]: audit 2024-07-28T19:45:19.437535+0000 mon.a (mon.0) 1222 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/1274339179"}]: dispatch 2024-07-28T19:45:20.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:19 smithi138 bash[67894]: cluster 2024-07-28T19:45:18.694631+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-07-28T19:45:20.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:19 smithi138 bash[67894]: audit 2024-07-28T19:45:19.437535+0000 mon.a (mon.0) 1222 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/1274339179"}]: dispatch 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:0.00% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:1.56% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:3.12% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:4.69% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:6.25% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:7.81% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:9.38% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:10.94% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:12.50% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:14.06% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:15.62% done 2024-07-28T19:45:20.137 INFO:tasks.workunit.client.0.smithi100.stdout:17.19% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:18.75% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:20.31% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:21.88% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:23.44% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:25.00% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:26.56% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:28.12% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:29.69% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:31.25% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:32.81% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:34.38% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:35.94% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:37.50% done 2024-07-28T19:45:20.138 INFO:tasks.workunit.client.0.smithi100.stdout:39.06% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:40.62% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:42.19% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:43.75% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:45.31% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:46.88% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:48.44% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:50.00% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:51.56% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:53.12% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:54.69% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:56.25% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:57.81% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:59.38% done 2024-07-28T19:45:20.139 INFO:tasks.workunit.client.0.smithi100.stdout:60.94% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:62.50% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:64.06% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:65.62% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:67.19% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:68.75% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:70.31% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:71.88% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:73.44% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:75.00% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:76.56% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:78.12% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:79.69% done 2024-07-28T19:45:20.140 INFO:tasks.workunit.client.0.smithi100.stdout:81.25% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:82.81% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:84.38% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:85.94% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:87.50% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:89.06% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:90.62% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:92.19% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:93.75% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:95.31% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:96.88% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:98.44% done 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (144 ms) 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-07-28T19:45:20.141 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:20.158 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:20.467 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestTrashPurge (330 ms) 2024-07-28T19:45:20.467 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-07-28T19:45:20.467 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:20.584 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (117 ms) 2024-07-28T19:45:20.584 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-07-28T19:45:20.584 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:20.644 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestListWatchers (60 ms) 2024-07-28T19:45:20.644 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-07-28T19:45:20.645 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:20.829 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestSetSnapById (185 ms) 2024-07-28T19:45:20.829 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.Namespaces 2024-07-28T19:45:20.877 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.Namespaces (47 ms) 2024-07-28T19:45:20.877 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-07-28T19:45:20.950 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:20.963 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:20.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:20 smithi100 bash[80199]: audit 2024-07-28T19:45:19.694798+0000 mon.a (mon.0) 1223 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/1274339179"}]': finished 2024-07-28T19:45:20.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:20 smithi100 bash[80199]: cluster 2024-07-28T19:45:19.694949+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-07-28T19:45:20.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:20 smithi100 bash[80199]: cluster 2024-07-28T19:45:19.902870+0000 mgr.y (mgr.25358) 382 : cluster [DBG] pgmap v746: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 190 KiB/s rd, 75 KiB/s wr, 176 op/s 2024-07-28T19:45:20.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:20 smithi100 bash[82644]: audit 2024-07-28T19:45:19.694798+0000 mon.a (mon.0) 1223 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/1274339179"}]': finished 2024-07-28T19:45:20.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:20 smithi100 bash[82644]: cluster 2024-07-28T19:45:19.694949+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-07-28T19:45:20.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:20 smithi100 bash[82644]: cluster 2024-07-28T19:45:19.902870+0000 mgr.y (mgr.25358) 382 : cluster [DBG] pgmap v746: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 190 KiB/s rd, 75 KiB/s wr, 176 op/s 2024-07-28T19:45:21.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:20 smithi138 bash[67894]: audit 2024-07-28T19:45:19.694798+0000 mon.a (mon.0) 1223 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.100:0/1274339179"}]': finished 2024-07-28T19:45:21.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:20 smithi138 bash[67894]: cluster 2024-07-28T19:45:19.694949+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-07-28T19:45:21.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:20 smithi138 bash[67894]: cluster 2024-07-28T19:45:19.902870+0000 mgr.y (mgr.25358) 382 : cluster [DBG] pgmap v746: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 190 KiB/s rd, 75 KiB/s wr, 176 op/s 2024-07-28T19:45:21.160 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.NamespacesPP (284 ms) 2024-07-28T19:45:21.161 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.Migration 2024-07-28T19:45:21.161 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:21.161 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:22.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:21 smithi138 bash[67894]: cluster 2024-07-28T19:45:20.762461+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-07-28T19:45:22.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:21 smithi100 bash[80199]: cluster 2024-07-28T19:45:20.762461+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-07-28T19:45:22.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:21 smithi100 bash[82644]: cluster 2024-07-28T19:45:20.762461+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-07-28T19:45:22.319 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.Migration (1158 ms) 2024-07-28T19:45:22.319 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-07-28T19:45:22.319 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:22.319 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:22.765 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:45:22 smithi138 bash[60615]: ts=2024-07-28T19:45:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:45:23.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:22 smithi138 bash[67894]: cluster 2024-07-28T19:45:21.904084+0000 mgr.y (mgr.25358) 383 : cluster [DBG] pgmap v748: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 132 KiB/s wr, 206 op/s 2024-07-28T19:45:23.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:22 smithi138 bash[67894]: audit 2024-07-28T19:45:22.639196+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:23.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:22 smithi100 bash[80199]: cluster 2024-07-28T19:45:21.904084+0000 mgr.y (mgr.25358) 383 : cluster [DBG] pgmap v748: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 132 KiB/s wr, 206 op/s 2024-07-28T19:45:23.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:22 smithi100 bash[80199]: audit 2024-07-28T19:45:22.639196+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:22 smithi100 bash[82644]: cluster 2024-07-28T19:45:21.904084+0000 mgr.y (mgr.25358) 383 : cluster [DBG] pgmap v748: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 132 KiB/s wr, 206 op/s 2024-07-28T19:45:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:22 smithi100 bash[82644]: audit 2024-07-28T19:45:22.639196+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:23.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:45:22] "GET /metrics HTTP/1.1" 200 75160 "" "Prometheus/2.43.0" 2024-07-28T19:45:23.445 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.MigrationPP (1126 ms) 2024-07-28T19:45:23.445 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-07-28T19:45:23.445 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:23.487 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (43 ms) 2024-07-28T19:45:23.488 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-07-28T19:45:23.488 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:23.533 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (46 ms) 2024-07-28T19:45:23.534 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-07-28T19:45:23.534 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:24.010 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (477 ms) 2024-07-28T19:45:24.011 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-07-28T19:45:24.078 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.PoolMetadata (68 ms) 2024-07-28T19:45:24.078 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-07-28T19:45:24.119 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.PoolMetadataPP (41 ms) 2024-07-28T19:45:24.119 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.Config 2024-07-28T19:45:24.129 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:24.202 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.Config (82 ms) 2024-07-28T19:45:24.202 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-07-28T19:45:24.210 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:24.293 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ConfigPP (92 ms) 2024-07-28T19:45:24.294 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-07-28T19:45:24.370 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[73305]: [28/Jul/2024:19:45:24] ENGINE Bus STOPPING 2024-07-28T19:45:24.370 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[73305]: [28/Jul/2024:19:45:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:45:24.370 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[73305]: [28/Jul/2024:19:45:24] ENGINE Bus STOPPED 2024-07-28T19:45:24.748 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[73305]: [28/Jul/2024:19:45:24] ENGINE Bus STARTING 2024-07-28T19:45:24.748 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[73305]: [28/Jul/2024:19:45:24] ENGINE Serving on http://:::9283 2024-07-28T19:45:24.748 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[73305]: [28/Jul/2024:19:45:24] ENGINE Bus STARTED 2024-07-28T19:45:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:24 smithi138 bash[67894]: cluster 2024-07-28T19:45:23.775261+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-07-28T19:45:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:24 smithi138 bash[67894]: audit 2024-07-28T19:45:24.046470+0000 mon.a (mon.0) 1228 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:45:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:24 smithi138 bash[67894]: audit 2024-07-28T19:45:24.072163+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:45:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:24 smithi138 bash[67894]: audit 2024-07-28T19:45:24.074963+0000 mon.a (mon.0) 1230 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:45:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:24 smithi138 bash[67894]: audit 2024-07-28T19:45:24.076254+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:45:25.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:24 smithi138 bash[67894]: audit 2024-07-28T19:45:24.089515+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:25.193 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:25.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[80199]: cluster 2024-07-28T19:45:23.775261+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-07-28T19:45:25.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[80199]: audit 2024-07-28T19:45:24.046470+0000 mon.a (mon.0) 1228 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:45:25.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[80199]: audit 2024-07-28T19:45:24.072163+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:45:25.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[80199]: audit 2024-07-28T19:45:24.074963+0000 mon.a (mon.0) 1230 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:45:25.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[80199]: audit 2024-07-28T19:45:24.076254+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:45:25.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[80199]: audit 2024-07-28T19:45:24.089515+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:25.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[82644]: cluster 2024-07-28T19:45:23.775261+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-07-28T19:45:25.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[82644]: audit 2024-07-28T19:45:24.046470+0000 mon.a (mon.0) 1228 : audit [INF] from='client.? 172.21.15.100:0/3960526926' entity='client.admin' 2024-07-28T19:45:25.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[82644]: audit 2024-07-28T19:45:24.072163+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:45:25.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[82644]: audit 2024-07-28T19:45:24.074963+0000 mon.a (mon.0) 1230 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:45:25.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[82644]: audit 2024-07-28T19:45:24.076254+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:45:25.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:24 smithi100 bash[82644]: audit 2024-07-28T19:45:24.089515+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:26.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:25 smithi138 bash[67894]: cluster 2024-07-28T19:45:23.905863+0000 mgr.y (mgr.25358) 384 : cluster [DBG] pgmap v750: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 547 KiB/s rd, 124 KiB/s wr, 659 op/s 2024-07-28T19:45:26.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:25 smithi138 bash[67894]: cluster 2024-07-28T19:45:25.095560+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-07-28T19:45:26.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:25 smithi138 bash[67894]: audit 2024-07-28T19:45:25.105871+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? 172.21.15.100:0/2195177830' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:26.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:25 smithi138 bash[67894]: audit 2024-07-28T19:45:25.181577+0000 mon.a (mon.0) 1235 : audit [INF] from='client.? 172.21.15.100:0/2195177830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:26.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:25 smithi138 bash[67894]: cluster 2024-07-28T19:45:25.183525+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-07-28T19:45:26.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[80199]: cluster 2024-07-28T19:45:23.905863+0000 mgr.y (mgr.25358) 384 : cluster [DBG] pgmap v750: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 547 KiB/s rd, 124 KiB/s wr, 659 op/s 2024-07-28T19:45:26.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[80199]: cluster 2024-07-28T19:45:25.095560+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-07-28T19:45:26.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[80199]: audit 2024-07-28T19:45:25.105871+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? 172.21.15.100:0/2195177830' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:26.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[80199]: audit 2024-07-28T19:45:25.181577+0000 mon.a (mon.0) 1235 : audit [INF] from='client.? 172.21.15.100:0/2195177830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:26.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[80199]: cluster 2024-07-28T19:45:25.183525+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-07-28T19:45:26.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[82644]: cluster 2024-07-28T19:45:23.905863+0000 mgr.y (mgr.25358) 384 : cluster [DBG] pgmap v750: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 547 KiB/s rd, 124 KiB/s wr, 659 op/s 2024-07-28T19:45:26.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[82644]: cluster 2024-07-28T19:45:25.095560+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-07-28T19:45:26.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[82644]: audit 2024-07-28T19:45:25.105871+0000 mon.a (mon.0) 1234 : audit [INF] from='client.? 172.21.15.100:0/2195177830' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:26.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[82644]: audit 2024-07-28T19:45:25.181577+0000 mon.a (mon.0) 1235 : audit [INF] from='client.? 172.21.15.100:0/2195177830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:26.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:25 smithi100 bash[82644]: cluster 2024-07-28T19:45:25.183525+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-07-28T19:45:27.275 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:27.294 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:27.324 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:27 smithi100 bash[80199]: cluster 2024-07-28T19:45:25.907030+0000 mgr.y (mgr.25358) 385 : cluster [DBG] pgmap v753: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 578 KiB/s rd, 37 KiB/s wr, 774 op/s 2024-07-28T19:45:27.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:27 smithi100 bash[80199]: cluster 2024-07-28T19:45:26.183963+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-07-28T19:45:27.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:27 smithi100 bash[82644]: cluster 2024-07-28T19:45:25.907030+0000 mgr.y (mgr.25358) 385 : cluster [DBG] pgmap v753: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 578 KiB/s rd, 37 KiB/s wr, 774 op/s 2024-07-28T19:45:27.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:27 smithi100 bash[82644]: cluster 2024-07-28T19:45:26.183963+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-07-28T19:45:27.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:27 smithi138 bash[67894]: cluster 2024-07-28T19:45:25.907030+0000 mgr.y (mgr.25358) 385 : cluster [DBG] pgmap v753: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 578 KiB/s rd, 37 KiB/s wr, 774 op/s 2024-07-28T19:45:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:27 smithi138 bash[67894]: cluster 2024-07-28T19:45:26.183963+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-07-28T19:45:28.391 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.PoolStatsPP (4098 ms) 2024-07-28T19:45:28.391 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-07-28T19:45:28.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:28 smithi100 bash[80199]: cluster 2024-07-28T19:45:27.195564+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-07-28T19:45:28.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:28 smithi100 bash[82644]: cluster 2024-07-28T19:45:27.195564+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-07-28T19:45:28.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:28 smithi138 bash[67894]: cluster 2024-07-28T19:45:27.195564+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-07-28T19:45:29.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:29 smithi100 bash[80199]: cluster 2024-07-28T19:45:27.908958+0000 mgr.y (mgr.25358) 386 : cluster [DBG] pgmap v756: 649 pgs: 27 unknown, 622 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 294 KiB/s rd, 28 KiB/s wr, 402 op/s 2024-07-28T19:45:29.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:29 smithi100 bash[80199]: cluster 2024-07-28T19:45:28.213933+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-07-28T19:45:29.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:29 smithi100 bash[82644]: cluster 2024-07-28T19:45:27.908958+0000 mgr.y (mgr.25358) 386 : cluster [DBG] pgmap v756: 649 pgs: 27 unknown, 622 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 294 KiB/s rd, 28 KiB/s wr, 402 op/s 2024-07-28T19:45:29.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:29 smithi100 bash[82644]: cluster 2024-07-28T19:45:28.213933+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-07-28T19:45:29.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:29 smithi138 bash[67894]: cluster 2024-07-28T19:45:27.908958+0000 mgr.y (mgr.25358) 386 : cluster [DBG] pgmap v756: 649 pgs: 27 unknown, 622 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 294 KiB/s rd, 28 KiB/s wr, 402 op/s 2024-07-28T19:45:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:29 smithi138 bash[67894]: cluster 2024-07-28T19:45:28.213933+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-07-28T19:45:30.223 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:30.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:30 smithi138 bash[67894]: cluster 2024-07-28T19:45:29.214145+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-07-28T19:45:30.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:30 smithi138 bash[67894]: audit 2024-07-28T19:45:29.218623+0000 mon.a (mon.0) 1241 : audit [INF] from='client.? 172.21.15.100:0/2818939895' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:30.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:30 smithi138 bash[67894]: audit 2024-07-28T19:45:30.216094+0000 mon.a (mon.0) 1242 : audit [INF] from='client.? 172.21.15.100:0/2818939895' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:30.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:30 smithi138 bash[67894]: cluster 2024-07-28T19:45:30.216549+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-07-28T19:45:30.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:30 smithi100 bash[80199]: cluster 2024-07-28T19:45:29.214145+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-07-28T19:45:30.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:30 smithi100 bash[80199]: audit 2024-07-28T19:45:29.218623+0000 mon.a (mon.0) 1241 : audit [INF] from='client.? 172.21.15.100:0/2818939895' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:30.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:30 smithi100 bash[80199]: audit 2024-07-28T19:45:30.216094+0000 mon.a (mon.0) 1242 : audit [INF] from='client.? 172.21.15.100:0/2818939895' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:30.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:30 smithi100 bash[80199]: cluster 2024-07-28T19:45:30.216549+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-07-28T19:45:30.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:30 smithi100 bash[82644]: cluster 2024-07-28T19:45:29.214145+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-07-28T19:45:30.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:30 smithi100 bash[82644]: audit 2024-07-28T19:45:29.218623+0000 mon.a (mon.0) 1241 : audit [INF] from='client.? 172.21.15.100:0/2818939895' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:30.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:30 smithi100 bash[82644]: audit 2024-07-28T19:45:30.216094+0000 mon.a (mon.0) 1242 : audit [INF] from='client.? 172.21.15.100:0/2818939895' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:30.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:30 smithi100 bash[82644]: cluster 2024-07-28T19:45:30.216549+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-07-28T19:45:31.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:31 smithi138 bash[67894]: cluster 2024-07-28T19:45:29.910835+0000 mgr.y (mgr.25358) 387 : cluster [DBG] pgmap v759: 681 pgs: 5 creating+peering, 29 unknown, 647 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 15 KiB/s wr, 217 op/s 2024-07-28T19:45:31.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:31 smithi138 bash[67894]: cluster 2024-07-28T19:45:30.221863+0000 mon.a (mon.0) 1244 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:45:31.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:31 smithi138 bash[67894]: cluster 2024-07-28T19:45:31.240686+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-07-28T19:45:31.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:31 smithi100 bash[80199]: cluster 2024-07-28T19:45:29.910835+0000 mgr.y (mgr.25358) 387 : cluster [DBG] pgmap v759: 681 pgs: 5 creating+peering, 29 unknown, 647 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 15 KiB/s wr, 217 op/s 2024-07-28T19:45:31.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:31 smithi100 bash[80199]: cluster 2024-07-28T19:45:30.221863+0000 mon.a (mon.0) 1244 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:45:31.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:31 smithi100 bash[80199]: cluster 2024-07-28T19:45:31.240686+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-07-28T19:45:31.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:31 smithi100 bash[82644]: cluster 2024-07-28T19:45:29.910835+0000 mgr.y (mgr.25358) 387 : cluster [DBG] pgmap v759: 681 pgs: 5 creating+peering, 29 unknown, 647 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 15 KiB/s wr, 217 op/s 2024-07-28T19:45:31.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:31 smithi100 bash[82644]: cluster 2024-07-28T19:45:30.221863+0000 mon.a (mon.0) 1244 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:45:31.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:31 smithi100 bash[82644]: cluster 2024-07-28T19:45:31.240686+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-07-28T19:45:32.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:32 smithi100 bash[80199]: cluster 2024-07-28T19:45:32.257406+0000 mon.a (mon.0) 1246 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:45:32.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:32 smithi100 bash[80199]: cluster 2024-07-28T19:45:32.272797+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-07-28T19:45:32.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:32 smithi100 bash[82644]: cluster 2024-07-28T19:45:32.257406+0000 mon.a (mon.0) 1246 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:45:32.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:32 smithi100 bash[82644]: cluster 2024-07-28T19:45:32.272797+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-07-28T19:45:32.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:32 smithi138 bash[67894]: cluster 2024-07-28T19:45:32.257406+0000 mon.a (mon.0) 1246 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:45:32.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:32 smithi138 bash[67894]: cluster 2024-07-28T19:45:32.272797+0000 mon.a (mon.0) 1247 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-07-28T19:45:32.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:45:32 smithi138 bash[60615]: ts=2024-07-28T19:45:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:45:33.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:45:32] "GET /metrics HTTP/1.1" 200 75160 "" "Prometheus/2.43.0" 2024-07-28T19:45:33.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:33 smithi100 bash[80199]: cluster 2024-07-28T19:45:31.911944+0000 mgr.y (mgr.25358) 388 : cluster [DBG] pgmap v762: 681 pgs: 11 creating+peering, 21 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 7.7 KiB/s wr, 112 op/s 2024-07-28T19:45:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:33 smithi100 bash[82644]: cluster 2024-07-28T19:45:31.911944+0000 mgr.y (mgr.25358) 388 : cluster [DBG] pgmap v762: 681 pgs: 11 creating+peering, 21 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 7.7 KiB/s wr, 112 op/s 2024-07-28T19:45:33.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:33 smithi138 bash[67894]: cluster 2024-07-28T19:45:31.911944+0000 mgr.y (mgr.25358) 388 : cluster [DBG] pgmap v762: 681 pgs: 11 creating+peering, 21 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 7.7 KiB/s wr, 112 op/s 2024-07-28T19:45:34.703 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ImageSpec (6312 ms) 2024-07-28T19:45:34.704 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-07-28T19:45:34.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:34 smithi100 bash[82644]: cluster 2024-07-28T19:45:33.332866+0000 mon.a (mon.0) 1248 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-07-28T19:45:34.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:34 smithi100 bash[80199]: cluster 2024-07-28T19:45:33.332866+0000 mon.a (mon.0) 1248 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-07-28T19:45:34.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:34 smithi138 bash[67894]: cluster 2024-07-28T19:45:33.332866+0000 mon.a (mon.0) 1248 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-07-28T19:45:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[80199]: cluster 2024-07-28T19:45:33.915223+0000 mgr.y (mgr.25358) 389 : cluster [DBG] pgmap v765: 681 pgs: 11 creating+peering, 5 unknown, 665 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.0 KiB/s wr, 11 op/s 2024-07-28T19:45:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[80199]: cluster 2024-07-28T19:45:34.344132+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-07-28T19:45:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[80199]: cluster 2024-07-28T19:45:35.189173+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-07-28T19:45:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[80199]: audit 2024-07-28T19:45:35.195141+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.100:0/2809856502' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[80199]: audit 2024-07-28T19:45:35.195568+0000 mon.a (mon.0) 1251 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[82644]: cluster 2024-07-28T19:45:33.915223+0000 mgr.y (mgr.25358) 389 : cluster [DBG] pgmap v765: 681 pgs: 11 creating+peering, 5 unknown, 665 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.0 KiB/s wr, 11 op/s 2024-07-28T19:45:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[82644]: cluster 2024-07-28T19:45:34.344132+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-07-28T19:45:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[82644]: cluster 2024-07-28T19:45:35.189173+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-07-28T19:45:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[82644]: audit 2024-07-28T19:45:35.195141+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.100:0/2809856502' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:35 smithi100 bash[82644]: audit 2024-07-28T19:45:35.195568+0000 mon.a (mon.0) 1251 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:35 smithi138 bash[67894]: cluster 2024-07-28T19:45:33.915223+0000 mgr.y (mgr.25358) 389 : cluster [DBG] pgmap v765: 681 pgs: 11 creating+peering, 5 unknown, 665 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.0 KiB/s wr, 11 op/s 2024-07-28T19:45:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:35 smithi138 bash[67894]: cluster 2024-07-28T19:45:34.344132+0000 mon.a (mon.0) 1249 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-07-28T19:45:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:35 smithi138 bash[67894]: cluster 2024-07-28T19:45:35.189173+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-07-28T19:45:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:35 smithi138 bash[67894]: audit 2024-07-28T19:45:35.195141+0000 mon.c (mon.2) 21 : audit [INF] from='client.? 172.21.15.100:0/2809856502' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:35 smithi138 bash[67894]: audit 2024-07-28T19:45:35.195568+0000 mon.a (mon.0) 1251 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:36.207 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:37 smithi100 bash[80199]: cluster 2024-07-28T19:45:35.916498+0000 mgr.y (mgr.25358) 390 : cluster [DBG] pgmap v768: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 5.5 KiB/s wr, 107 op/s 2024-07-28T19:45:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:37 smithi100 bash[80199]: audit 2024-07-28T19:45:36.197209+0000 mon.a (mon.0) 1252 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:37 smithi100 bash[80199]: cluster 2024-07-28T19:45:36.197758+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-07-28T19:45:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:37 smithi100 bash[80199]: cluster 2024-07-28T19:45:36.337672+0000 mon.a (mon.0) 1254 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:45:37.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:37 smithi100 bash[82644]: cluster 2024-07-28T19:45:35.916498+0000 mgr.y (mgr.25358) 390 : cluster [DBG] pgmap v768: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 5.5 KiB/s wr, 107 op/s 2024-07-28T19:45:37.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:37 smithi100 bash[82644]: audit 2024-07-28T19:45:36.197209+0000 mon.a (mon.0) 1252 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:37.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:37 smithi100 bash[82644]: cluster 2024-07-28T19:45:36.197758+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-07-28T19:45:37.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:37 smithi100 bash[82644]: cluster 2024-07-28T19:45:36.337672+0000 mon.a (mon.0) 1254 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:45:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:37 smithi138 bash[67894]: cluster 2024-07-28T19:45:35.916498+0000 mgr.y (mgr.25358) 390 : cluster [DBG] pgmap v768: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 5.5 KiB/s wr, 107 op/s 2024-07-28T19:45:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:37 smithi138 bash[67894]: audit 2024-07-28T19:45:36.197209+0000 mon.a (mon.0) 1252 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:37 smithi138 bash[67894]: cluster 2024-07-28T19:45:36.197758+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-07-28T19:45:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:37 smithi138 bash[67894]: cluster 2024-07-28T19:45:36.337672+0000 mon.a (mon.0) 1254 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:45:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:38 smithi100 bash[82644]: cluster 2024-07-28T19:45:37.199464+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-07-28T19:45:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:38 smithi100 bash[82644]: audit 2024-07-28T19:45:37.646659+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:38 smithi100 bash[82644]: audit 2024-07-28T19:45:37.648662+0000 mon.a (mon.0) 1257 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:38.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:38 smithi100 bash[80199]: cluster 2024-07-28T19:45:37.199464+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-07-28T19:45:38.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:38 smithi100 bash[80199]: audit 2024-07-28T19:45:37.646659+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:38.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:38 smithi100 bash[80199]: audit 2024-07-28T19:45:37.648662+0000 mon.a (mon.0) 1257 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:38.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:38 smithi138 bash[67894]: cluster 2024-07-28T19:45:37.199464+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-07-28T19:45:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:38 smithi138 bash[67894]: audit 2024-07-28T19:45:37.646659+0000 mon.a (mon.0) 1256 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:38 smithi138 bash[67894]: audit 2024-07-28T19:45:37.648662+0000 mon.a (mon.0) 1257 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:39 smithi100 bash[80199]: cluster 2024-07-28T19:45:37.918527+0000 mgr.y (mgr.25358) 391 : cluster [DBG] pgmap v771: 713 pgs: 29 unknown, 684 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 3.5 KiB/s wr, 95 op/s 2024-07-28T19:45:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:39 smithi100 bash[80199]: cluster 2024-07-28T19:45:38.221772+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-07-28T19:45:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:39 smithi100 bash[80199]: audit 2024-07-28T19:45:38.474344+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:39 smithi100 bash[80199]: cluster 2024-07-28T19:45:38.641690+0000 mon.a (mon.0) 1260 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:45:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:39 smithi100 bash[82644]: cluster 2024-07-28T19:45:37.918527+0000 mgr.y (mgr.25358) 391 : cluster [DBG] pgmap v771: 713 pgs: 29 unknown, 684 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 3.5 KiB/s wr, 95 op/s 2024-07-28T19:45:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:39 smithi100 bash[82644]: cluster 2024-07-28T19:45:38.221772+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-07-28T19:45:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:39 smithi100 bash[82644]: audit 2024-07-28T19:45:38.474344+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:39 smithi100 bash[82644]: cluster 2024-07-28T19:45:38.641690+0000 mon.a (mon.0) 1260 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:45:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:39 smithi138 bash[67894]: cluster 2024-07-28T19:45:37.918527+0000 mgr.y (mgr.25358) 391 : cluster [DBG] pgmap v771: 713 pgs: 29 unknown, 684 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 3.5 KiB/s wr, 95 op/s 2024-07-28T19:45:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:39 smithi138 bash[67894]: cluster 2024-07-28T19:45:38.221772+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-07-28T19:45:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:39 smithi138 bash[67894]: audit 2024-07-28T19:45:38.474344+0000 mon.a (mon.0) 1259 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:39 smithi138 bash[67894]: cluster 2024-07-28T19:45:38.641690+0000 mon.a (mon.0) 1260 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:45:40.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:40 smithi138 bash[67894]: cluster 2024-07-28T19:45:39.230125+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-07-28T19:45:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:40 smithi138 bash[67894]: cluster 2024-07-28T19:45:40.197592+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-07-28T19:45:40.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:40 smithi100 bash[80199]: cluster 2024-07-28T19:45:39.230125+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-07-28T19:45:40.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:40 smithi100 bash[80199]: cluster 2024-07-28T19:45:40.197592+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-07-28T19:45:40.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:40 smithi100 bash[82644]: cluster 2024-07-28T19:45:39.230125+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-07-28T19:45:40.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:40 smithi100 bash[82644]: cluster 2024-07-28T19:45:40.197592+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-07-28T19:45:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:41 smithi138 bash[67894]: cluster 2024-07-28T19:45:39.920361+0000 mgr.y (mgr.25358) 392 : cluster [DBG] pgmap v774: 713 pgs: 6 unknown, 707 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 2.2 KiB/s wr, 74 op/s 2024-07-28T19:45:41.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:41 smithi100 bash[82644]: cluster 2024-07-28T19:45:39.920361+0000 mgr.y (mgr.25358) 392 : cluster [DBG] pgmap v774: 713 pgs: 6 unknown, 707 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 2.2 KiB/s wr, 74 op/s 2024-07-28T19:45:41.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:41 smithi100 bash[80199]: cluster 2024-07-28T19:45:39.920361+0000 mgr.y (mgr.25358) 392 : cluster [DBG] pgmap v774: 713 pgs: 6 unknown, 707 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 2.2 KiB/s wr, 74 op/s 2024-07-28T19:45:42.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:42 smithi138 bash[67894]: cluster 2024-07-28T19:45:41.264062+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-07-28T19:45:42.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:45:42 smithi138 bash[60615]: ts=2024-07-28T19:45:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:45:42.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:42 smithi100 bash[80199]: cluster 2024-07-28T19:45:41.264062+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-07-28T19:45:42.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:42 smithi100 bash[82644]: cluster 2024-07-28T19:45:41.264062+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-07-28T19:45:43.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:45:42] "GET /metrics HTTP/1.1" 200 82325 "" "Prometheus/2.43.0" 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 3 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 3 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout:child: image202 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout:child: image204 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout:child: image203 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-14/image202/85fbe4fa4a3e 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-33/image203/85fb85c5c1e9 2024-07-28T19:45:43.358 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:45:43.359 INFO:tasks.workunit.client.0.smithi100.stdout:found child test-librbd-smithi100-68802-14/image204/85fb503946c7 2024-07-28T19:45:43.359 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:45:43.359 INFO:tasks.workunit.client.0.smithi100.stdout:num children is: 0 2024-07-28T19:45:43.359 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:45:43.359 INFO:tasks.workunit.client.0.smithi100.stdout:num snaps is: 0 2024-07-28T19:45:43.359 INFO:tasks.workunit.client.0.smithi100.stdout:expected: 0 2024-07-28T19:45:43.359 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8654 ms) 2024-07-28T19:45:43.359 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-07-28T19:45:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:43 smithi138 bash[67894]: cluster 2024-07-28T19:45:41.921525+0000 mgr.y (mgr.25358) 393 : cluster [DBG] pgmap v777: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:45:43.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:43 smithi138 bash[67894]: cluster 2024-07-28T19:45:42.286939+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-07-28T19:45:43.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:43 smithi100 bash[80199]: cluster 2024-07-28T19:45:41.921525+0000 mgr.y (mgr.25358) 393 : cluster [DBG] pgmap v777: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:45:43.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:43 smithi100 bash[80199]: cluster 2024-07-28T19:45:42.286939+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-07-28T19:45:43.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:43 smithi100 bash[82644]: cluster 2024-07-28T19:45:41.921525+0000 mgr.y (mgr.25358) 393 : cluster [DBG] pgmap v777: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:45:43.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:43 smithi100 bash[82644]: cluster 2024-07-28T19:45:42.286939+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-07-28T19:45:44.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:44 smithi100 bash[80199]: cluster 2024-07-28T19:45:43.294225+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-07-28T19:45:44.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:44 smithi100 bash[82644]: cluster 2024-07-28T19:45:43.294225+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-07-28T19:45:44.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:44 smithi138 bash[67894]: cluster 2024-07-28T19:45:43.294225+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-07-28T19:45:45.225 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:45.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[80199]: cluster 2024-07-28T19:45:43.924682+0000 mgr.y (mgr.25358) 394 : cluster [DBG] pgmap v780: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:45:45.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[80199]: cluster 2024-07-28T19:45:44.322863+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-07-28T19:45:45.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[80199]: audit 2024-07-28T19:45:44.326063+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.100:0/2662702239' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:45.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[80199]: audit 2024-07-28T19:45:44.327091+0000 mon.a (mon.0) 1267 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:45.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[80199]: audit 2024-07-28T19:45:45.201782+0000 mon.a (mon.0) 1268 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:45.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[80199]: cluster 2024-07-28T19:45:45.202407+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-07-28T19:45:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[82644]: cluster 2024-07-28T19:45:43.924682+0000 mgr.y (mgr.25358) 394 : cluster [DBG] pgmap v780: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:45:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[82644]: cluster 2024-07-28T19:45:44.322863+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-07-28T19:45:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[82644]: audit 2024-07-28T19:45:44.326063+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.100:0/2662702239' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[82644]: audit 2024-07-28T19:45:44.327091+0000 mon.a (mon.0) 1267 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[82644]: audit 2024-07-28T19:45:45.201782+0000 mon.a (mon.0) 1268 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:45 smithi100 bash[82644]: cluster 2024-07-28T19:45:45.202407+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-07-28T19:45:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:45 smithi138 bash[67894]: cluster 2024-07-28T19:45:43.924682+0000 mgr.y (mgr.25358) 394 : cluster [DBG] pgmap v780: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:45:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:45 smithi138 bash[67894]: cluster 2024-07-28T19:45:44.322863+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-07-28T19:45:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:45 smithi138 bash[67894]: audit 2024-07-28T19:45:44.326063+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.100:0/2662702239' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:45 smithi138 bash[67894]: audit 2024-07-28T19:45:44.327091+0000 mon.a (mon.0) 1267 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:45:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:45 smithi138 bash[67894]: audit 2024-07-28T19:45:45.201782+0000 mon.a (mon.0) 1268 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:45:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:45 smithi138 bash[67894]: cluster 2024-07-28T19:45:45.202407+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-07-28T19:45:47.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:47 smithi100 bash[80199]: cluster 2024-07-28T19:45:45.925930+0000 mgr.y (mgr.25358) 395 : cluster [DBG] pgmap v783: 713 pgs: 10 creating+peering, 22 unknown, 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 8.5 KiB/s wr, 146 op/s 2024-07-28T19:45:47.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:47 smithi100 bash[80199]: cluster 2024-07-28T19:45:46.201402+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-07-28T19:45:47.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:47 smithi100 bash[82644]: cluster 2024-07-28T19:45:45.925930+0000 mgr.y (mgr.25358) 395 : cluster [DBG] pgmap v783: 713 pgs: 10 creating+peering, 22 unknown, 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 8.5 KiB/s wr, 146 op/s 2024-07-28T19:45:47.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:47 smithi100 bash[82644]: cluster 2024-07-28T19:45:46.201402+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-07-28T19:45:47.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:47 smithi138 bash[67894]: cluster 2024-07-28T19:45:45.925930+0000 mgr.y (mgr.25358) 395 : cluster [DBG] pgmap v783: 713 pgs: 10 creating+peering, 22 unknown, 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 8.5 KiB/s wr, 146 op/s 2024-07-28T19:45:47.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:47 smithi138 bash[67894]: cluster 2024-07-28T19:45:46.201402+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-07-28T19:45:48.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:48 smithi100 bash[80199]: cluster 2024-07-28T19:45:47.212209+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-07-28T19:45:48.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:48 smithi100 bash[82644]: cluster 2024-07-28T19:45:47.212209+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-07-28T19:45:48.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:48 smithi138 bash[67894]: cluster 2024-07-28T19:45:47.212209+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-07-28T19:45:49.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:49 smithi100 bash[80199]: cluster 2024-07-28T19:45:47.927948+0000 mgr.y (mgr.25358) 396 : cluster [DBG] pgmap v786: 713 pgs: 10 creating+peering, 16 unknown, 687 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 145 op/s 2024-07-28T19:45:49.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:49 smithi100 bash[80199]: cluster 2024-07-28T19:45:48.227775+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-07-28T19:45:49.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:49 smithi100 bash[80199]: audit 2024-07-28T19:45:48.545620+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:49.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:49 smithi100 bash[82644]: cluster 2024-07-28T19:45:47.927948+0000 mgr.y (mgr.25358) 396 : cluster [DBG] pgmap v786: 713 pgs: 10 creating+peering, 16 unknown, 687 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 145 op/s 2024-07-28T19:45:49.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:49 smithi100 bash[82644]: cluster 2024-07-28T19:45:48.227775+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-07-28T19:45:49.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:49 smithi100 bash[82644]: audit 2024-07-28T19:45:48.545620+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:49.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:49 smithi138 bash[67894]: cluster 2024-07-28T19:45:47.927948+0000 mgr.y (mgr.25358) 396 : cluster [DBG] pgmap v786: 713 pgs: 10 creating+peering, 16 unknown, 687 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 145 op/s 2024-07-28T19:45:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:49 smithi138 bash[67894]: cluster 2024-07-28T19:45:48.227775+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-07-28T19:45:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:49 smithi138 bash[67894]: audit 2024-07-28T19:45:48.545620+0000 mon.a (mon.0) 1273 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:50.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:50 smithi100 bash[80199]: cluster 2024-07-28T19:45:49.227803+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-07-28T19:45:50.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:50 smithi100 bash[80199]: cluster 2024-07-28T19:45:50.201249+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-07-28T19:45:50.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:50 smithi100 bash[82644]: cluster 2024-07-28T19:45:49.227803+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-07-28T19:45:50.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:50 smithi100 bash[82644]: cluster 2024-07-28T19:45:50.201249+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-07-28T19:45:50.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:50 smithi138 bash[67894]: cluster 2024-07-28T19:45:49.227803+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-07-28T19:45:50.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:50 smithi138 bash[67894]: cluster 2024-07-28T19:45:50.201249+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-07-28T19:45:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:51 smithi138 bash[67894]: cluster 2024-07-28T19:45:49.929924+0000 mgr.y (mgr.25358) 397 : cluster [DBG] pgmap v789: 713 pgs: 6 creating+peering, 707 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 1.5 KiB/s wr, 37 op/s 2024-07-28T19:45:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:51 smithi100 bash[80199]: cluster 2024-07-28T19:45:49.929924+0000 mgr.y (mgr.25358) 397 : cluster [DBG] pgmap v789: 713 pgs: 6 creating+peering, 707 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 1.5 KiB/s wr, 37 op/s 2024-07-28T19:45:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:51 smithi100 bash[82644]: cluster 2024-07-28T19:45:49.929924+0000 mgr.y (mgr.25358) 397 : cluster [DBG] pgmap v789: 713 pgs: 6 creating+peering, 707 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 1.5 KiB/s wr, 37 op/s 2024-07-28T19:45:52.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:52 smithi138 bash[67894]: cluster 2024-07-28T19:45:51.273840+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-07-28T19:45:52.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:45:52 smithi138 bash[60615]: ts=2024-07-28T19:45:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:45:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:52 smithi100 bash[80199]: cluster 2024-07-28T19:45:51.273840+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-07-28T19:45:52.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:52 smithi100 bash[82644]: cluster 2024-07-28T19:45:51.273840+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-07-28T19:45:53.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:45:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:45:52] "GET /metrics HTTP/1.1" 200 82335 "" "Prometheus/2.43.0" 2024-07-28T19:45:53.508 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.QuiesceWatch (10150 ms) 2024-07-28T19:45:53.508 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-07-28T19:45:53.509 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:53.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:53 smithi138 bash[67894]: cluster 2024-07-28T19:45:51.931205+0000 mgr.y (mgr.25358) 398 : cluster [DBG] pgmap v792: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-07-28T19:45:53.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:53 smithi138 bash[67894]: cluster 2024-07-28T19:45:52.284000+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-07-28T19:45:53.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:53 smithi138 bash[67894]: audit 2024-07-28T19:45:52.646776+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:53.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:53 smithi138 bash[67894]: audit 2024-07-28T19:45:52.649009+0000 mon.a (mon.0) 1279 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:53 smithi100 bash[80199]: cluster 2024-07-28T19:45:51.931205+0000 mgr.y (mgr.25358) 398 : cluster [DBG] pgmap v792: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-07-28T19:45:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:53 smithi100 bash[80199]: cluster 2024-07-28T19:45:52.284000+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-07-28T19:45:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:53 smithi100 bash[80199]: audit 2024-07-28T19:45:52.646776+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:53 smithi100 bash[80199]: audit 2024-07-28T19:45:52.649009+0000 mon.a (mon.0) 1279 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:53 smithi100 bash[82644]: cluster 2024-07-28T19:45:51.931205+0000 mgr.y (mgr.25358) 398 : cluster [DBG] pgmap v792: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 KiB/s wr, 7 op/s 2024-07-28T19:45:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:53 smithi100 bash[82644]: cluster 2024-07-28T19:45:52.284000+0000 mon.a (mon.0) 1277 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-07-28T19:45:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:53 smithi100 bash[82644]: audit 2024-07-28T19:45:52.646776+0000 mon.a (mon.0) 1278 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:53 smithi100 bash[82644]: audit 2024-07-28T19:45:52.649009+0000 mon.a (mon.0) 1279 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:45:54.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:54 smithi138 bash[67894]: cluster 2024-07-28T19:45:53.284819+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-07-28T19:45:54.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:54 smithi100 bash[80199]: cluster 2024-07-28T19:45:53.284819+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-07-28T19:45:54.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:54 smithi100 bash[82644]: cluster 2024-07-28T19:45:53.284819+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-07-28T19:45:55.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:55 smithi138 bash[67894]: cluster 2024-07-28T19:45:53.934542+0000 mgr.y (mgr.25358) 399 : cluster [DBG] pgmap v795: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.5 KiB/s wr, 76 op/s 2024-07-28T19:45:55.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:55 smithi138 bash[67894]: cluster 2024-07-28T19:45:54.308019+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-07-28T19:45:55.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:55 smithi138 bash[67894]: cluster 2024-07-28T19:45:55.206463+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-07-28T19:45:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:55 smithi100 bash[80199]: cluster 2024-07-28T19:45:53.934542+0000 mgr.y (mgr.25358) 399 : cluster [DBG] pgmap v795: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.5 KiB/s wr, 76 op/s 2024-07-28T19:45:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:55 smithi100 bash[80199]: cluster 2024-07-28T19:45:54.308019+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-07-28T19:45:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:55 smithi100 bash[80199]: cluster 2024-07-28T19:45:55.206463+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-07-28T19:45:55.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:55 smithi100 bash[82644]: cluster 2024-07-28T19:45:53.934542+0000 mgr.y (mgr.25358) 399 : cluster [DBG] pgmap v795: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.5 KiB/s wr, 76 op/s 2024-07-28T19:45:55.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:55 smithi100 bash[82644]: cluster 2024-07-28T19:45:54.308019+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-07-28T19:45:55.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:55 smithi100 bash[82644]: cluster 2024-07-28T19:45:55.206463+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-07-28T19:45:57.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:57 smithi100 bash[80199]: cluster 2024-07-28T19:45:55.935759+0000 mgr.y (mgr.25358) 400 : cluster [DBG] pgmap v798: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 6.2 KiB/s wr, 105 op/s 2024-07-28T19:45:57.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:57 smithi100 bash[80199]: cluster 2024-07-28T19:45:56.208393+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-07-28T19:45:57.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:57 smithi100 bash[82644]: cluster 2024-07-28T19:45:55.935759+0000 mgr.y (mgr.25358) 400 : cluster [DBG] pgmap v798: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 6.2 KiB/s wr, 105 op/s 2024-07-28T19:45:57.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:57 smithi100 bash[82644]: cluster 2024-07-28T19:45:56.208393+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-07-28T19:45:57.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:57 smithi138 bash[67894]: cluster 2024-07-28T19:45:55.935759+0000 mgr.y (mgr.25358) 400 : cluster [DBG] pgmap v798: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 6.2 KiB/s wr, 105 op/s 2024-07-28T19:45:57.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:57 smithi138 bash[67894]: cluster 2024-07-28T19:45:56.208393+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-07-28T19:45:58.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:58 smithi100 bash[82644]: cluster 2024-07-28T19:45:57.221073+0000 mon.a (mon.0) 1284 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-07-28T19:45:58.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:58 smithi100 bash[80199]: cluster 2024-07-28T19:45:57.221073+0000 mon.a (mon.0) 1284 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-07-28T19:45:58.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:58 smithi138 bash[67894]: cluster 2024-07-28T19:45:57.221073+0000 mon.a (mon.0) 1284 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-07-28T19:45:59.491 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (5983 ms) 2024-07-28T19:45:59.491 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-07-28T19:45:59.491 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:45:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:59 smithi100 bash[82644]: cluster 2024-07-28T19:45:57.937831+0000 mgr.y (mgr.25358) 401 : cluster [DBG] pgmap v801: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 2.5 KiB/s wr, 65 op/s 2024-07-28T19:45:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:59 smithi100 bash[82644]: cluster 2024-07-28T19:45:58.230340+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-07-28T19:45:59.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:45:59 smithi100 bash[82644]: audit 2024-07-28T19:45:58.633840+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:59.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:59 smithi100 bash[80199]: cluster 2024-07-28T19:45:57.937831+0000 mgr.y (mgr.25358) 401 : cluster [DBG] pgmap v801: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 2.5 KiB/s wr, 65 op/s 2024-07-28T19:45:59.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:59 smithi100 bash[80199]: cluster 2024-07-28T19:45:58.230340+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-07-28T19:45:59.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:45:59 smithi100 bash[80199]: audit 2024-07-28T19:45:58.633840+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:45:59.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:59 smithi138 bash[67894]: cluster 2024-07-28T19:45:57.937831+0000 mgr.y (mgr.25358) 401 : cluster [DBG] pgmap v801: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 2.5 KiB/s wr, 65 op/s 2024-07-28T19:45:59.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:59 smithi138 bash[67894]: cluster 2024-07-28T19:45:58.230340+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-07-28T19:45:59.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:45:59 smithi138 bash[67894]: audit 2024-07-28T19:45:58.633840+0000 mon.a (mon.0) 1286 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:00.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:00 smithi138 bash[67894]: cluster 2024-07-28T19:45:59.247121+0000 mon.a (mon.0) 1287 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-07-28T19:46:00.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:00 smithi138 bash[67894]: cluster 2024-07-28T19:46:00.206679+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-07-28T19:46:00.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:00 smithi100 bash[82644]: cluster 2024-07-28T19:45:59.247121+0000 mon.a (mon.0) 1287 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-07-28T19:46:00.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:00 smithi100 bash[82644]: cluster 2024-07-28T19:46:00.206679+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-07-28T19:46:00.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:00 smithi100 bash[80199]: cluster 2024-07-28T19:45:59.247121+0000 mon.a (mon.0) 1287 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-07-28T19:46:00.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:00 smithi100 bash[80199]: cluster 2024-07-28T19:46:00.206679+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-07-28T19:46:01.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:01 smithi138 bash[67894]: cluster 2024-07-28T19:45:59.939747+0000 mgr.y (mgr.25358) 402 : cluster [DBG] pgmap v804: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-07-28T19:46:01.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:01 smithi100 bash[80199]: cluster 2024-07-28T19:45:59.939747+0000 mgr.y (mgr.25358) 402 : cluster [DBG] pgmap v804: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-07-28T19:46:01.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:01 smithi100 bash[82644]: cluster 2024-07-28T19:45:59.939747+0000 mgr.y (mgr.25358) 402 : cluster [DBG] pgmap v804: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-07-28T19:46:02.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:02 smithi138 bash[67894]: cluster 2024-07-28T19:46:01.277701+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-07-28T19:46:02.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:46:02 smithi138 bash[60615]: ts=2024-07-28T19:46:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:46:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:02 smithi100 bash[80199]: cluster 2024-07-28T19:46:01.277701+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-07-28T19:46:02.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:02 smithi100 bash[82644]: cluster 2024-07-28T19:46:01.277701+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-07-28T19:46:03.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:46:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:46:02] "GET /metrics HTTP/1.1" 200 82335 "" "Prometheus/2.43.0" 2024-07-28T19:46:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:03 smithi138 bash[67894]: cluster 2024-07-28T19:46:01.940943+0000 mgr.y (mgr.25358) 403 : cluster [DBG] pgmap v807: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-07-28T19:46:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:03 smithi138 bash[67894]: cluster 2024-07-28T19:46:02.288457+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-07-28T19:46:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:03 smithi100 bash[80199]: cluster 2024-07-28T19:46:01.940943+0000 mgr.y (mgr.25358) 403 : cluster [DBG] pgmap v807: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-07-28T19:46:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:03 smithi100 bash[80199]: cluster 2024-07-28T19:46:02.288457+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-07-28T19:46:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:03 smithi100 bash[82644]: cluster 2024-07-28T19:46:01.940943+0000 mgr.y (mgr.25358) 403 : cluster [DBG] pgmap v807: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-07-28T19:46:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:03 smithi100 bash[82644]: cluster 2024-07-28T19:46:02.288457+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-07-28T19:46:04.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:04 smithi100 bash[80199]: cluster 2024-07-28T19:46:03.294349+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-07-28T19:46:04.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:04 smithi100 bash[82644]: cluster 2024-07-28T19:46:03.294349+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-07-28T19:46:04.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:04 smithi138 bash[67894]: cluster 2024-07-28T19:46:03.294349+0000 mon.a (mon.0) 1291 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-07-28T19:46:05.448 INFO:tasks.workunit.client.0.smithi100.stdout:10.00% done 2024-07-28T19:46:05.448 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.QuiesceWatchError (5957 ms) 2024-07-28T19:46:05.448 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-07-28T19:46:05.448 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:46:05.488 INFO:tasks.workunit.client.0.smithi100.stderr:test quiesce is not long enough to time out 2024-07-28T19:46:05.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:05 smithi100 bash[80199]: cluster 2024-07-28T19:46:03.944316+0000 mgr.y (mgr.25358) 404 : cluster [DBG] pgmap v810: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 2.0 KiB/s wr, 22 op/s 2024-07-28T19:46:05.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:05 smithi100 bash[80199]: cluster 2024-07-28T19:46:04.316436+0000 mon.a (mon.0) 1292 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-07-28T19:46:05.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:05 smithi100 bash[80199]: cluster 2024-07-28T19:46:05.211406+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-07-28T19:46:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:05 smithi100 bash[82644]: cluster 2024-07-28T19:46:03.944316+0000 mgr.y (mgr.25358) 404 : cluster [DBG] pgmap v810: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 2.0 KiB/s wr, 22 op/s 2024-07-28T19:46:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:05 smithi100 bash[82644]: cluster 2024-07-28T19:46:04.316436+0000 mon.a (mon.0) 1292 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-07-28T19:46:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:05 smithi100 bash[82644]: cluster 2024-07-28T19:46:05.211406+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-07-28T19:46:05.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:05 smithi138 bash[67894]: cluster 2024-07-28T19:46:03.944316+0000 mgr.y (mgr.25358) 404 : cluster [DBG] pgmap v810: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 2.0 KiB/s wr, 22 op/s 2024-07-28T19:46:05.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:05 smithi138 bash[67894]: cluster 2024-07-28T19:46:04.316436+0000 mon.a (mon.0) 1292 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-07-28T19:46:05.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:05 smithi138 bash[67894]: cluster 2024-07-28T19:46:05.211406+0000 mon.a (mon.0) 1293 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-07-28T19:46:07.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:07 smithi100 bash[80199]: cluster 2024-07-28T19:46:05.945540+0000 mgr.y (mgr.25358) 405 : cluster [DBG] pgmap v813: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 6.7 KiB/s wr, 100 op/s 2024-07-28T19:46:07.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:07 smithi100 bash[82644]: cluster 2024-07-28T19:46:05.945540+0000 mgr.y (mgr.25358) 405 : cluster [DBG] pgmap v813: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 6.7 KiB/s wr, 100 op/s 2024-07-28T19:46:07.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:07 smithi138 bash[67894]: cluster 2024-07-28T19:46:05.945540+0000 mgr.y (mgr.25358) 405 : cluster [DBG] pgmap v813: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 6.7 KiB/s wr, 100 op/s 2024-07-28T19:46:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:08 smithi100 bash[80199]: audit 2024-07-28T19:46:07.640450+0000 mon.a (mon.0) 1294 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:08.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:08 smithi100 bash[82644]: audit 2024-07-28T19:46:07.640450+0000 mon.a (mon.0) 1294 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:08.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:08 smithi138 bash[67894]: audit 2024-07-28T19:46:07.640450+0000 mon.a (mon.0) 1294 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:09.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:09 smithi100 bash[80199]: cluster 2024-07-28T19:46:07.947580+0000 mgr.y (mgr.25358) 406 : cluster [DBG] pgmap v814: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.8 KiB/s wr, 71 op/s 2024-07-28T19:46:09.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:09 smithi100 bash[82644]: cluster 2024-07-28T19:46:07.947580+0000 mgr.y (mgr.25358) 406 : cluster [DBG] pgmap v814: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.8 KiB/s wr, 71 op/s 2024-07-28T19:46:09.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:09 smithi138 bash[67894]: cluster 2024-07-28T19:46:07.947580+0000 mgr.y (mgr.25358) 406 : cluster [DBG] pgmap v814: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.8 KiB/s wr, 71 op/s 2024-07-28T19:46:11.501 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:11 smithi100 bash[80199]: cluster 2024-07-28T19:46:09.952289+0000 mgr.y (mgr.25358) 407 : cluster [DBG] pgmap v815: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 89 op/s 2024-07-28T19:46:11.502 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:11 smithi100 bash[80199]: cluster 2024-07-28T19:46:10.212905+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-07-28T19:46:11.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:11 smithi100 bash[82644]: cluster 2024-07-28T19:46:09.952289+0000 mgr.y (mgr.25358) 407 : cluster [DBG] pgmap v815: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 89 op/s 2024-07-28T19:46:11.503 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:11 smithi100 bash[82644]: cluster 2024-07-28T19:46:10.212905+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-07-28T19:46:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:11 smithi138 bash[67894]: cluster 2024-07-28T19:46:09.952289+0000 mgr.y (mgr.25358) 407 : cluster [DBG] pgmap v815: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 89 op/s 2024-07-28T19:46:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:11 smithi138 bash[67894]: cluster 2024-07-28T19:46:10.212905+0000 mon.a (mon.0) 1295 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-07-28T19:46:12.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:46:12 smithi138 bash[60615]: ts=2024-07-28T19:46:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:46:13.212 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:46:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:46:12] "GET /metrics HTTP/1.1" 200 82338 "" "Prometheus/2.43.0" 2024-07-28T19:46:13.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:13 smithi100 bash[80199]: cluster 2024-07-28T19:46:11.953508+0000 mgr.y (mgr.25358) 408 : cluster [DBG] pgmap v817: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 3.5 KiB/s wr, 67 op/s 2024-07-28T19:46:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:13 smithi100 bash[82644]: cluster 2024-07-28T19:46:11.953508+0000 mgr.y (mgr.25358) 408 : cluster [DBG] pgmap v817: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 3.5 KiB/s wr, 67 op/s 2024-07-28T19:46:13.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:13 smithi138 bash[67894]: cluster 2024-07-28T19:46:11.953508+0000 mgr.y (mgr.25358) 408 : cluster [DBG] pgmap v817: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 3.5 KiB/s wr, 67 op/s 2024-07-28T19:46:14.291 INFO:tasks.workunit.client.0.smithi100.stderr:test quiesce is timed out 2024-07-28T19:46:14.300 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 0 2024-07-28T19:46:15.301 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 1 2024-07-28T19:46:15.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:15 smithi100 bash[80199]: cluster 2024-07-28T19:46:13.957554+0000 mgr.y (mgr.25358) 409 : cluster [DBG] pgmap v818: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 936 B/s wr, 22 op/s 2024-07-28T19:46:15.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:15 smithi100 bash[80199]: cluster 2024-07-28T19:46:14.229729+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-07-28T19:46:15.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:15 smithi100 bash[80199]: cluster 2024-07-28T19:46:15.216608+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-07-28T19:46:15.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:15 smithi100 bash[82644]: cluster 2024-07-28T19:46:13.957554+0000 mgr.y (mgr.25358) 409 : cluster [DBG] pgmap v818: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 936 B/s wr, 22 op/s 2024-07-28T19:46:15.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:15 smithi100 bash[82644]: cluster 2024-07-28T19:46:14.229729+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-07-28T19:46:15.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:15 smithi100 bash[82644]: cluster 2024-07-28T19:46:15.216608+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-07-28T19:46:15.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:15 smithi138 bash[67894]: cluster 2024-07-28T19:46:13.957554+0000 mgr.y (mgr.25358) 409 : cluster [DBG] pgmap v818: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 936 B/s wr, 22 op/s 2024-07-28T19:46:15.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:15 smithi138 bash[67894]: cluster 2024-07-28T19:46:14.229729+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-07-28T19:46:15.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:15 smithi138 bash[67894]: cluster 2024-07-28T19:46:15.216608+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-07-28T19:46:16.301 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 2 2024-07-28T19:46:17.301 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 3 2024-07-28T19:46:17.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:17 smithi100 bash[80199]: cluster 2024-07-28T19:46:15.958678+0000 mgr.y (mgr.25358) 410 : cluster [DBG] pgmap v821: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 681 B/s wr, 6 op/s 2024-07-28T19:46:17.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:17 smithi100 bash[82644]: cluster 2024-07-28T19:46:15.958678+0000 mgr.y (mgr.25358) 410 : cluster [DBG] pgmap v821: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 681 B/s wr, 6 op/s 2024-07-28T19:46:17.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:17 smithi138 bash[67894]: cluster 2024-07-28T19:46:15.958678+0000 mgr.y (mgr.25358) 410 : cluster [DBG] pgmap v821: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 681 B/s wr, 6 op/s 2024-07-28T19:46:18.301 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 4 2024-07-28T19:46:19.301 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 5 2024-07-28T19:46:19.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:19 smithi100 bash[82644]: cluster 2024-07-28T19:46:17.960675+0000 mgr.y (mgr.25358) 411 : cluster [DBG] pgmap v822: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 396 B/s wr, 4 op/s 2024-07-28T19:46:19.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:19 smithi100 bash[80199]: cluster 2024-07-28T19:46:17.960675+0000 mgr.y (mgr.25358) 411 : cluster [DBG] pgmap v822: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 396 B/s wr, 4 op/s 2024-07-28T19:46:19.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:19 smithi138 bash[67894]: cluster 2024-07-28T19:46:17.960675+0000 mgr.y (mgr.25358) 411 : cluster [DBG] pgmap v822: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 396 B/s wr, 4 op/s 2024-07-28T19:46:20.301 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 6 2024-07-28T19:46:21.301 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 7 2024-07-28T19:46:21.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:21 smithi100 bash[82644]: cluster 2024-07-28T19:46:19.965399+0000 mgr.y (mgr.25358) 412 : cluster [DBG] pgmap v823: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-07-28T19:46:21.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:21 smithi100 bash[80199]: cluster 2024-07-28T19:46:19.965399+0000 mgr.y (mgr.25358) 412 : cluster [DBG] pgmap v823: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-07-28T19:46:21.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:21 smithi138 bash[67894]: cluster 2024-07-28T19:46:19.965399+0000 mgr.y (mgr.25358) 412 : cluster [DBG] pgmap v823: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-07-28T19:46:22.301 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 8 2024-07-28T19:46:22.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:46:22 smithi138 bash[60615]: ts=2024-07-28T19:46:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:46:23.245 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:46:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:46:22] "GET /metrics HTTP/1.1" 200 82348 "" "Prometheus/2.43.0" 2024-07-28T19:46:23.302 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 9 2024-07-28T19:46:23.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:23 smithi100 bash[80199]: cluster 2024-07-28T19:46:21.967383+0000 mgr.y (mgr.25358) 413 : cluster [DBG] pgmap v824: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-07-28T19:46:23.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:23 smithi100 bash[80199]: audit 2024-07-28T19:46:22.640798+0000 mon.a (mon.0) 1298 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:23.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:23 smithi100 bash[82644]: cluster 2024-07-28T19:46:21.967383+0000 mgr.y (mgr.25358) 413 : cluster [DBG] pgmap v824: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-07-28T19:46:23.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:23 smithi100 bash[82644]: audit 2024-07-28T19:46:22.640798+0000 mon.a (mon.0) 1298 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:23.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:23 smithi138 bash[67894]: cluster 2024-07-28T19:46:21.967383+0000 mgr.y (mgr.25358) 413 : cluster [DBG] pgmap v824: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-07-28T19:46:23.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:23 smithi138 bash[67894]: audit 2024-07-28T19:46:22.640798+0000 mon.a (mon.0) 1298 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:24.302 INFO:tasks.workunit.client.0.smithi100.stderr:waiting for timed out ... 10 2024-07-28T19:46:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:24 smithi138 bash[67894]: audit 2024-07-28T19:46:24.220188+0000 mon.a (mon.0) 1299 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:46:24.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:24 smithi100 bash[80199]: audit 2024-07-28T19:46:24.220188+0000 mon.a (mon.0) 1299 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:46:24.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:24 smithi100 bash[82644]: audit 2024-07-28T19:46:24.220188+0000 mon.a (mon.0) 1299 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:46:25.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:25 smithi138 bash[67894]: cluster 2024-07-28T19:46:23.971494+0000 mgr.y (mgr.25358) 414 : cluster [DBG] pgmap v825: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 315 B/s wr, 3 op/s 2024-07-28T19:46:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:25 smithi100 bash[82644]: cluster 2024-07-28T19:46:23.971494+0000 mgr.y (mgr.25358) 414 : cluster [DBG] pgmap v825: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 315 B/s wr, 3 op/s 2024-07-28T19:46:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:25 smithi100 bash[80199]: cluster 2024-07-28T19:46:23.971494+0000 mgr.y (mgr.25358) 414 : cluster [DBG] pgmap v825: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 315 B/s wr, 3 op/s 2024-07-28T19:46:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:27 smithi138 bash[67894]: cluster 2024-07-28T19:46:25.974736+0000 mgr.y (mgr.25358) 415 : cluster [DBG] pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 95 B/s rd, 0 op/s 2024-07-28T19:46:27.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:27 smithi100 bash[80199]: cluster 2024-07-28T19:46:25.974736+0000 mgr.y (mgr.25358) 415 : cluster [DBG] pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 95 B/s rd, 0 op/s 2024-07-28T19:46:27.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:27 smithi100 bash[82644]: cluster 2024-07-28T19:46:25.974736+0000 mgr.y (mgr.25358) 415 : cluster [DBG] pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 95 B/s rd, 0 op/s 2024-07-28T19:46:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:29 smithi138 bash[67894]: cluster 2024-07-28T19:46:27.976735+0000 mgr.y (mgr.25358) 416 : cluster [DBG] pgmap v827: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-07-28T19:46:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:29 smithi100 bash[80199]: cluster 2024-07-28T19:46:27.976735+0000 mgr.y (mgr.25358) 416 : cluster [DBG] pgmap v827: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-07-28T19:46:29.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:29 smithi100 bash[82644]: cluster 2024-07-28T19:46:27.976735+0000 mgr.y (mgr.25358) 416 : cluster [DBG] pgmap v827: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-07-28T19:46:31.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[82644]: audit 2024-07-28T19:46:29.831419+0000 mon.a (mon.0) 1300 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[82644]: audit 2024-07-28T19:46:29.839027+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[82644]: cluster 2024-07-28T19:46:29.981270+0000 mgr.y (mgr.25358) 417 : cluster [DBG] pgmap v828: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[82644]: audit 2024-07-28T19:46:30.101405+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[82644]: audit 2024-07-28T19:46:30.108243+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[82644]: audit 2024-07-28T19:46:30.525831+0000 mon.a (mon.0) 1304 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[82644]: audit 2024-07-28T19:46:30.527320+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[82644]: audit 2024-07-28T19:46:30.539291+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[80199]: audit 2024-07-28T19:46:29.831419+0000 mon.a (mon.0) 1300 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[80199]: audit 2024-07-28T19:46:29.839027+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[80199]: cluster 2024-07-28T19:46:29.981270+0000 mgr.y (mgr.25358) 417 : cluster [DBG] pgmap v828: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-07-28T19:46:31.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[80199]: audit 2024-07-28T19:46:30.101405+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[80199]: audit 2024-07-28T19:46:30.108243+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[80199]: audit 2024-07-28T19:46:30.525831+0000 mon.a (mon.0) 1304 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:46:31.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[80199]: audit 2024-07-28T19:46:30.527320+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:46:31.250 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:30 smithi100 bash[80199]: audit 2024-07-28T19:46:30.539291+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:30 smithi138 bash[67894]: audit 2024-07-28T19:46:29.831419+0000 mon.a (mon.0) 1300 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:30 smithi138 bash[67894]: audit 2024-07-28T19:46:29.839027+0000 mon.a (mon.0) 1301 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:30 smithi138 bash[67894]: cluster 2024-07-28T19:46:29.981270+0000 mgr.y (mgr.25358) 417 : cluster [DBG] pgmap v828: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-07-28T19:46:31.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:30 smithi138 bash[67894]: audit 2024-07-28T19:46:30.101405+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:30 smithi138 bash[67894]: audit 2024-07-28T19:46:30.108243+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:31.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:30 smithi138 bash[67894]: audit 2024-07-28T19:46:30.525831+0000 mon.a (mon.0) 1304 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:46:31.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:30 smithi138 bash[67894]: audit 2024-07-28T19:46:30.527320+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:46:31.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:30 smithi138 bash[67894]: audit 2024-07-28T19:46:30.539291+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:46:32.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:46:32 smithi138 bash[60615]: ts=2024-07-28T19:46:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:46:33.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:46:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:46:32] "GET /metrics HTTP/1.1" 200 82348 "" "Prometheus/2.43.0" 2024-07-28T19:46:33.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:33 smithi100 bash[82644]: cluster 2024-07-28T19:46:31.983132+0000 mgr.y (mgr.25358) 418 : cluster [DBG] pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:33.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:33 smithi100 bash[80199]: cluster 2024-07-28T19:46:31.983132+0000 mgr.y (mgr.25358) 418 : cluster [DBG] pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:33.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:33 smithi138 bash[67894]: cluster 2024-07-28T19:46:31.983132+0000 mgr.y (mgr.25358) 418 : cluster [DBG] pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:35.302 INFO:tasks.workunit.client.0.smithi100.stderr:test retry succeeds 2024-07-28T19:46:35.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:35 smithi138 bash[67894]: cluster 2024-07-28T19:46:33.987233+0000 mgr.y (mgr.25358) 419 : cluster [DBG] pgmap v830: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:35.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:35 smithi100 bash[80199]: cluster 2024-07-28T19:46:33.987233+0000 mgr.y (mgr.25358) 419 : cluster [DBG] pgmap v830: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:35.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:35 smithi100 bash[82644]: cluster 2024-07-28T19:46:33.987233+0000 mgr.y (mgr.25358) 419 : cluster [DBG] pgmap v830: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:37.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:37 smithi100 bash[82644]: cluster 2024-07-28T19:46:35.990457+0000 mgr.y (mgr.25358) 420 : cluster [DBG] pgmap v831: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:37.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:37 smithi100 bash[82644]: cluster 2024-07-28T19:46:36.062059+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-07-28T19:46:37.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:37 smithi100 bash[80199]: cluster 2024-07-28T19:46:35.990457+0000 mgr.y (mgr.25358) 420 : cluster [DBG] pgmap v831: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:37.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:37 smithi100 bash[80199]: cluster 2024-07-28T19:46:36.062059+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-07-28T19:46:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:37 smithi138 bash[67894]: cluster 2024-07-28T19:46:35.990457+0000 mgr.y (mgr.25358) 420 : cluster [DBG] pgmap v831: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:46:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:37 smithi138 bash[67894]: cluster 2024-07-28T19:46:36.062059+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-07-28T19:46:38.288 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32840 ms) 2024-07-28T19:46:38.288 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-07-28T19:46:38.288 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:46:38.438 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.WriteZeroes (150 ms) 2024-07-28T19:46:38.438 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-07-28T19:46:38.438 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:46:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:38 smithi100 bash[80199]: cluster 2024-07-28T19:46:37.073788+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-07-28T19:46:38.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:38 smithi100 bash[80199]: audit 2024-07-28T19:46:37.641088+0000 mon.a (mon.0) 1309 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:38 smithi100 bash[82644]: cluster 2024-07-28T19:46:37.073788+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-07-28T19:46:38.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:38 smithi100 bash[82644]: audit 2024-07-28T19:46:37.641088+0000 mon.a (mon.0) 1309 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:38 smithi138 bash[67894]: cluster 2024-07-28T19:46:37.073788+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-07-28T19:46:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:38 smithi138 bash[67894]: audit 2024-07-28T19:46:37.641088+0000 mon.a (mon.0) 1309 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:38.584 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (146 ms) 2024-07-28T19:46:38.584 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-07-28T19:46:38.584 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:46:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:39 smithi100 bash[80199]: cluster 2024-07-28T19:46:37.992431+0000 mgr.y (mgr.25358) 421 : cluster [DBG] pgmap v834: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-07-28T19:46:39.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:39 smithi100 bash[80199]: cluster 2024-07-28T19:46:38.084114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-07-28T19:46:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:39 smithi100 bash[82644]: cluster 2024-07-28T19:46:37.992431+0000 mgr.y (mgr.25358) 421 : cluster [DBG] pgmap v834: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-07-28T19:46:39.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:39 smithi100 bash[82644]: cluster 2024-07-28T19:46:38.084114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-07-28T19:46:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:39 smithi138 bash[67894]: cluster 2024-07-28T19:46:37.992431+0000 mgr.y (mgr.25358) 421 : cluster [DBG] pgmap v834: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-07-28T19:46:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:39 smithi138 bash[67894]: cluster 2024-07-28T19:46:38.084114+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-07-28T19:46:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[82644]: cluster 2024-07-28T19:46:39.089916+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-07-28T19:46:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[82644]: audit 2024-07-28T19:46:39.245358+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2024-07-28T19:46:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[82644]: audit 2024-07-28T19:46:39.245862+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]: dispatch 2024-07-28T19:46:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[82644]: audit 2024-07-28T19:46:39.246304+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]: dispatch 2024-07-28T19:46:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[82644]: audit 2024-07-28T19:46:39.246735+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1c", "id": [0, 3]}]: dispatch 2024-07-28T19:46:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[82644]: audit 2024-07-28T19:46:39.247100+0000 mon.a (mon.0) 1316 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1", "id": [0, 1]}]: dispatch 2024-07-28T19:46:40.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[82644]: audit 2024-07-28T19:46:39.247444+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]: dispatch 2024-07-28T19:46:40.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[80199]: cluster 2024-07-28T19:46:39.089916+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-07-28T19:46:40.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[80199]: audit 2024-07-28T19:46:39.245358+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2024-07-28T19:46:40.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[80199]: audit 2024-07-28T19:46:39.245862+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]: dispatch 2024-07-28T19:46:40.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[80199]: audit 2024-07-28T19:46:39.246304+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]: dispatch 2024-07-28T19:46:40.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[80199]: audit 2024-07-28T19:46:39.246735+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1c", "id": [0, 3]}]: dispatch 2024-07-28T19:46:40.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[80199]: audit 2024-07-28T19:46:39.247100+0000 mon.a (mon.0) 1316 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1", "id": [0, 1]}]: dispatch 2024-07-28T19:46:40.500 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:40 smithi100 bash[80199]: audit 2024-07-28T19:46:39.247444+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]: dispatch 2024-07-28T19:46:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:40 smithi138 bash[67894]: cluster 2024-07-28T19:46:39.089916+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-07-28T19:46:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:40 smithi138 bash[67894]: audit 2024-07-28T19:46:39.245358+0000 mon.a (mon.0) 1312 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2024-07-28T19:46:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:40 smithi138 bash[67894]: audit 2024-07-28T19:46:39.245862+0000 mon.a (mon.0) 1313 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]: dispatch 2024-07-28T19:46:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:40 smithi138 bash[67894]: audit 2024-07-28T19:46:39.246304+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]: dispatch 2024-07-28T19:46:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:40 smithi138 bash[67894]: audit 2024-07-28T19:46:39.246735+0000 mon.a (mon.0) 1315 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1c", "id": [0, 3]}]: dispatch 2024-07-28T19:46:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:40 smithi138 bash[67894]: audit 2024-07-28T19:46:39.247100+0000 mon.a (mon.0) 1316 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1", "id": [0, 1]}]: dispatch 2024-07-28T19:46:40.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:40 smithi138 bash[67894]: audit 2024-07-28T19:46:39.247444+0000 mon.a (mon.0) 1317 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]: dispatch 2024-07-28T19:46:41.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[80199]: cluster 2024-07-28T19:46:39.995039+0000 mgr.y (mgr.25358) 422 : cluster [DBG] pgmap v837: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 5.7 KiB/s wr, 112 op/s 2024-07-28T19:46:41.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[80199]: audit 2024-07-28T19:46:40.113003+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]': finished 2024-07-28T19:46:41.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[80199]: audit 2024-07-28T19:46:40.113188+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]': finished 2024-07-28T19:46:41.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[80199]: audit 2024-07-28T19:46:40.113288+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-07-28T19:46:41.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[80199]: audit 2024-07-28T19:46:40.113443+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1c", "id": [0, 3]}]': finished 2024-07-28T19:46:41.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[80199]: audit 2024-07-28T19:46:40.113540+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1", "id": [0, 1]}]': finished 2024-07-28T19:46:41.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[80199]: audit 2024-07-28T19:46:40.113725+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]': finished 2024-07-28T19:46:41.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[80199]: cluster 2024-07-28T19:46:40.113822+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-07-28T19:46:41.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[82644]: cluster 2024-07-28T19:46:39.995039+0000 mgr.y (mgr.25358) 422 : cluster [DBG] pgmap v837: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 5.7 KiB/s wr, 112 op/s 2024-07-28T19:46:41.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[82644]: audit 2024-07-28T19:46:40.113003+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]': finished 2024-07-28T19:46:41.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[82644]: audit 2024-07-28T19:46:40.113188+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]': finished 2024-07-28T19:46:41.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[82644]: audit 2024-07-28T19:46:40.113288+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-07-28T19:46:41.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[82644]: audit 2024-07-28T19:46:40.113443+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1c", "id": [0, 3]}]': finished 2024-07-28T19:46:41.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[82644]: audit 2024-07-28T19:46:40.113540+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1", "id": [0, 1]}]': finished 2024-07-28T19:46:41.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[82644]: audit 2024-07-28T19:46:40.113725+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]': finished 2024-07-28T19:46:41.500 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:41 smithi100 bash[82644]: cluster 2024-07-28T19:46:40.113822+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-07-28T19:46:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:41 smithi138 bash[67894]: cluster 2024-07-28T19:46:39.995039+0000 mgr.y (mgr.25358) 422 : cluster [DBG] pgmap v837: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 5.7 KiB/s wr, 112 op/s 2024-07-28T19:46:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:41 smithi138 bash[67894]: audit 2024-07-28T19:46:40.113003+0000 mon.a (mon.0) 1318 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]': finished 2024-07-28T19:46:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:41 smithi138 bash[67894]: audit 2024-07-28T19:46:40.113188+0000 mon.a (mon.0) 1319 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]': finished 2024-07-28T19:46:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:41 smithi138 bash[67894]: audit 2024-07-28T19:46:40.113288+0000 mon.a (mon.0) 1320 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-07-28T19:46:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:41 smithi138 bash[67894]: audit 2024-07-28T19:46:40.113443+0000 mon.a (mon.0) 1321 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1c", "id": [0, 3]}]': finished 2024-07-28T19:46:41.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:41 smithi138 bash[67894]: audit 2024-07-28T19:46:40.113540+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1", "id": [0, 1]}]': finished 2024-07-28T19:46:41.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:41 smithi138 bash[67894]: audit 2024-07-28T19:46:40.113725+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]': finished 2024-07-28T19:46:41.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:41 smithi138 bash[67894]: cluster 2024-07-28T19:46:40.113822+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-07-28T19:46:42.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:42 smithi100 bash[82644]: cluster 2024-07-28T19:46:41.114999+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-07-28T19:46:42.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:42 smithi100 bash[80199]: cluster 2024-07-28T19:46:41.114999+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-07-28T19:46:42.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:42 smithi138 bash[67894]: cluster 2024-07-28T19:46:41.114999+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-07-28T19:46:42.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:46:42 smithi138 bash[60615]: ts=2024-07-28T19:46:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:46:43.147 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:46:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:46:42] "GET /metrics HTTP/1.1" 200 82347 "" "Prometheus/2.43.0" 2024-07-28T19:46:43.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:43 smithi100 bash[80199]: cluster 2024-07-28T19:46:41.996349+0000 mgr.y (mgr.25358) 423 : cluster [DBG] pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 7.0 KiB/s wr, 127 op/s 2024-07-28T19:46:43.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:43 smithi100 bash[80199]: cluster 2024-07-28T19:46:42.141650+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-07-28T19:46:43.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:43 smithi100 bash[82644]: cluster 2024-07-28T19:46:41.996349+0000 mgr.y (mgr.25358) 423 : cluster [DBG] pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 7.0 KiB/s wr, 127 op/s 2024-07-28T19:46:43.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:43 smithi100 bash[82644]: cluster 2024-07-28T19:46:42.141650+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-07-28T19:46:43.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:43 smithi138 bash[67894]: cluster 2024-07-28T19:46:41.996349+0000 mgr.y (mgr.25358) 423 : cluster [DBG] pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 7.0 KiB/s wr, 127 op/s 2024-07-28T19:46:43.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:43 smithi138 bash[67894]: cluster 2024-07-28T19:46:42.141650+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-07-28T19:46:44.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:44 smithi100 bash[80199]: cluster 2024-07-28T19:46:43.142551+0000 mon.a (mon.0) 1327 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-07-28T19:46:44.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:44 smithi100 bash[82644]: cluster 2024-07-28T19:46:43.142551+0000 mon.a (mon.0) 1327 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-07-28T19:46:44.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:44 smithi138 bash[67894]: cluster 2024-07-28T19:46:43.142551+0000 mon.a (mon.0) 1327 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-07-28T19:46:45.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:45 smithi100 bash[80199]: cluster 2024-07-28T19:46:43.999848+0000 mgr.y (mgr.25358) 424 : cluster [DBG] pgmap v843: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 7.2 KiB/s wr, 79 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-07-28T19:46:45.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:45 smithi100 bash[80199]: cluster 2024-07-28T19:46:44.164230+0000 mon.a (mon.0) 1328 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-07-28T19:46:45.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:45 smithi100 bash[82644]: cluster 2024-07-28T19:46:43.999848+0000 mgr.y (mgr.25358) 424 : cluster [DBG] pgmap v843: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 7.2 KiB/s wr, 79 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-07-28T19:46:45.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:45 smithi100 bash[82644]: cluster 2024-07-28T19:46:44.164230+0000 mon.a (mon.0) 1328 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-07-28T19:46:45.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:45 smithi138 bash[67894]: cluster 2024-07-28T19:46:43.999848+0000 mgr.y (mgr.25358) 424 : cluster [DBG] pgmap v843: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 7.2 KiB/s wr, 79 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-07-28T19:46:45.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:45 smithi138 bash[67894]: cluster 2024-07-28T19:46:44.164230+0000 mon.a (mon.0) 1328 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-07-28T19:46:46.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:46 smithi100 bash[80199]: cluster 2024-07-28T19:46:45.167385+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-07-28T19:46:46.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:46 smithi100 bash[80199]: cluster 2024-07-28T19:46:45.227980+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-07-28T19:46:46.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:46 smithi100 bash[82644]: cluster 2024-07-28T19:46:45.167385+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-07-28T19:46:46.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:46 smithi100 bash[82644]: cluster 2024-07-28T19:46:45.227980+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-07-28T19:46:46.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:46 smithi138 bash[67894]: cluster 2024-07-28T19:46:45.167385+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-07-28T19:46:46.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:46 smithi138 bash[67894]: cluster 2024-07-28T19:46:45.227980+0000 mon.a (mon.0) 1330 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-07-28T19:46:47.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:47 smithi100 bash[80199]: cluster 2024-07-28T19:46:46.001480+0000 mgr.y (mgr.25358) 425 : cluster [DBG] pgmap v847: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.2 KiB/s wr, 65 op/s; 0 B/s, 2 keys/s, 1 objects/s recovering 2024-07-28T19:46:47.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:47 smithi100 bash[80199]: cluster 2024-07-28T19:46:46.228221+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-07-28T19:46:47.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:47 smithi100 bash[82644]: cluster 2024-07-28T19:46:46.001480+0000 mgr.y (mgr.25358) 425 : cluster [DBG] pgmap v847: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.2 KiB/s wr, 65 op/s; 0 B/s, 2 keys/s, 1 objects/s recovering 2024-07-28T19:46:47.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:47 smithi100 bash[82644]: cluster 2024-07-28T19:46:46.228221+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-07-28T19:46:47.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:47 smithi138 bash[67894]: cluster 2024-07-28T19:46:46.001480+0000 mgr.y (mgr.25358) 425 : cluster [DBG] pgmap v847: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.2 KiB/s wr, 65 op/s; 0 B/s, 2 keys/s, 1 objects/s recovering 2024-07-28T19:46:47.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:47 smithi138 bash[67894]: cluster 2024-07-28T19:46:46.228221+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-07-28T19:46:48.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:48 smithi138 bash[67894]: cluster 2024-07-28T19:46:47.249334+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-07-28T19:46:48.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:48 smithi100 bash[80199]: cluster 2024-07-28T19:46:47.249334+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-07-28T19:46:48.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:48 smithi100 bash[82644]: cluster 2024-07-28T19:46:47.249334+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-07-28T19:46:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:49 smithi138 bash[67894]: cluster 2024-07-28T19:46:48.003574+0000 mgr.y (mgr.25358) 426 : cluster [DBG] pgmap v850: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 266 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:46:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:49 smithi138 bash[67894]: cluster 2024-07-28T19:46:48.268397+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-07-28T19:46:49.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:49 smithi100 bash[80199]: cluster 2024-07-28T19:46:48.003574+0000 mgr.y (mgr.25358) 426 : cluster [DBG] pgmap v850: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 266 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:46:49.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:49 smithi100 bash[80199]: cluster 2024-07-28T19:46:48.268397+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-07-28T19:46:49.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:49 smithi100 bash[82644]: cluster 2024-07-28T19:46:48.003574+0000 mgr.y (mgr.25358) 426 : cluster [DBG] pgmap v850: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 266 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2024-07-28T19:46:49.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:49 smithi100 bash[82644]: cluster 2024-07-28T19:46:48.268397+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-07-28T19:46:50.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:50 smithi138 bash[67894]: cluster 2024-07-28T19:46:49.277756+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-07-28T19:46:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:50 smithi100 bash[80199]: cluster 2024-07-28T19:46:49.277756+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-07-28T19:46:50.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:50 smithi100 bash[82644]: cluster 2024-07-28T19:46:49.277756+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-07-28T19:46:51.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:51 smithi138 bash[67894]: cluster 2024-07-28T19:46:50.005633+0000 mgr.y (mgr.25358) 427 : cluster [DBG] pgmap v853: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.2 KiB/s wr, 239 op/s 2024-07-28T19:46:51.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:51 smithi138 bash[67894]: cluster 2024-07-28T19:46:50.288131+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-07-28T19:46:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:51 smithi100 bash[80199]: cluster 2024-07-28T19:46:50.005633+0000 mgr.y (mgr.25358) 427 : cluster [DBG] pgmap v853: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.2 KiB/s wr, 239 op/s 2024-07-28T19:46:51.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:51 smithi100 bash[80199]: cluster 2024-07-28T19:46:50.288131+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-07-28T19:46:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:51 smithi100 bash[82644]: cluster 2024-07-28T19:46:50.005633+0000 mgr.y (mgr.25358) 427 : cluster [DBG] pgmap v853: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.2 KiB/s wr, 239 op/s 2024-07-28T19:46:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:51 smithi100 bash[82644]: cluster 2024-07-28T19:46:50.288131+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-07-28T19:46:52.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:52 smithi100 bash[80199]: cluster 2024-07-28T19:46:51.308388+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-07-28T19:46:52.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:52 smithi100 bash[82644]: cluster 2024-07-28T19:46:51.308388+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-07-28T19:46:52.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:52 smithi138 bash[67894]: cluster 2024-07-28T19:46:51.308388+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-07-28T19:46:52.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:46:52 smithi138 bash[60615]: ts=2024-07-28T19:46:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:46:53.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:46:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:46:52] "GET /metrics HTTP/1.1" 200 82354 "" "Prometheus/2.43.0" 2024-07-28T19:46:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:53 smithi100 bash[80199]: cluster 2024-07-28T19:46:52.006841+0000 mgr.y (mgr.25358) 428 : cluster [DBG] pgmap v856: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.2 KiB/s wr, 239 op/s 2024-07-28T19:46:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:53 smithi100 bash[80199]: cluster 2024-07-28T19:46:52.325328+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-07-28T19:46:53.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:53 smithi100 bash[80199]: audit 2024-07-28T19:46:52.641517+0000 mon.a (mon.0) 1338 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:53 smithi100 bash[82644]: cluster 2024-07-28T19:46:52.006841+0000 mgr.y (mgr.25358) 428 : cluster [DBG] pgmap v856: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.2 KiB/s wr, 239 op/s 2024-07-28T19:46:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:53 smithi100 bash[82644]: cluster 2024-07-28T19:46:52.325328+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-07-28T19:46:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:53 smithi100 bash[82644]: audit 2024-07-28T19:46:52.641517+0000 mon.a (mon.0) 1338 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:53 smithi138 bash[67894]: cluster 2024-07-28T19:46:52.006841+0000 mgr.y (mgr.25358) 428 : cluster [DBG] pgmap v856: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 4.2 KiB/s wr, 239 op/s 2024-07-28T19:46:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:53 smithi138 bash[67894]: cluster 2024-07-28T19:46:52.325328+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-07-28T19:46:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:53 smithi138 bash[67894]: audit 2024-07-28T19:46:52.641517+0000 mon.a (mon.0) 1338 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:46:54.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:54 smithi100 bash[80199]: cluster 2024-07-28T19:46:53.332144+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-07-28T19:46:54.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:54 smithi100 bash[82644]: cluster 2024-07-28T19:46:53.332144+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-07-28T19:46:54.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:54 smithi138 bash[67894]: cluster 2024-07-28T19:46:53.332144+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-07-28T19:46:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:55 smithi100 bash[80199]: cluster 2024-07-28T19:46:54.010189+0000 mgr.y (mgr.25358) 429 : cluster [DBG] pgmap v859: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-07-28T19:46:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:55 smithi100 bash[80199]: cluster 2024-07-28T19:46:54.349464+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-07-28T19:46:55.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:55 smithi100 bash[82644]: cluster 2024-07-28T19:46:54.010189+0000 mgr.y (mgr.25358) 429 : cluster [DBG] pgmap v859: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-07-28T19:46:55.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:55 smithi100 bash[82644]: cluster 2024-07-28T19:46:54.349464+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-07-28T19:46:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:55 smithi138 bash[67894]: cluster 2024-07-28T19:46:54.010189+0000 mgr.y (mgr.25358) 429 : cluster [DBG] pgmap v859: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-07-28T19:46:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:55 smithi138 bash[67894]: cluster 2024-07-28T19:46:54.349464+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-07-28T19:46:56.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:56 smithi100 bash[80199]: cluster 2024-07-28T19:46:55.364310+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-07-28T19:46:56.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:56 smithi100 bash[82644]: cluster 2024-07-28T19:46:55.364310+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-07-28T19:46:56.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:56 smithi138 bash[67894]: cluster 2024-07-28T19:46:55.364310+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-07-28T19:46:57.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:57 smithi100 bash[80199]: cluster 2024-07-28T19:46:56.011356+0000 mgr.y (mgr.25358) 430 : cluster [DBG] pgmap v862: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.0 KiB/s wr, 27 op/s 2024-07-28T19:46:57.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:57 smithi100 bash[80199]: cluster 2024-07-28T19:46:56.368462+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-07-28T19:46:57.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:57 smithi100 bash[82644]: cluster 2024-07-28T19:46:56.011356+0000 mgr.y (mgr.25358) 430 : cluster [DBG] pgmap v862: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.0 KiB/s wr, 27 op/s 2024-07-28T19:46:57.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:57 smithi100 bash[82644]: cluster 2024-07-28T19:46:56.368462+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-07-28T19:46:57.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:57 smithi138 bash[67894]: cluster 2024-07-28T19:46:56.011356+0000 mgr.y (mgr.25358) 430 : cluster [DBG] pgmap v862: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 5.0 KiB/s wr, 27 op/s 2024-07-28T19:46:57.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:57 smithi138 bash[67894]: cluster 2024-07-28T19:46:56.368462+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-07-28T19:46:58.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:58 smithi100 bash[80199]: cluster 2024-07-28T19:46:57.380526+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-07-28T19:46:58.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:58 smithi100 bash[82644]: cluster 2024-07-28T19:46:57.380526+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-07-28T19:46:58.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:58 smithi138 bash[67894]: cluster 2024-07-28T19:46:57.380526+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-07-28T19:46:59.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:59 smithi100 bash[80199]: cluster 2024-07-28T19:46:58.012189+0000 mgr.y (mgr.25358) 431 : cluster [DBG] pgmap v865: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1022 B/s rd, 0 B/s wr, 2 op/s 2024-07-28T19:46:59.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:46:59 smithi100 bash[80199]: cluster 2024-07-28T19:46:58.388741+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-07-28T19:46:59.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:59 smithi100 bash[82644]: cluster 2024-07-28T19:46:58.012189+0000 mgr.y (mgr.25358) 431 : cluster [DBG] pgmap v865: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1022 B/s rd, 0 B/s wr, 2 op/s 2024-07-28T19:46:59.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:46:59 smithi100 bash[82644]: cluster 2024-07-28T19:46:58.388741+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-07-28T19:46:59.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:59 smithi138 bash[67894]: cluster 2024-07-28T19:46:58.012189+0000 mgr.y (mgr.25358) 431 : cluster [DBG] pgmap v865: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1022 B/s rd, 0 B/s wr, 2 op/s 2024-07-28T19:46:59.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:46:59 smithi138 bash[67894]: cluster 2024-07-28T19:46:58.388741+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-07-28T19:47:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:00 smithi100 bash[80199]: cluster 2024-07-28T19:46:59.395439+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-07-28T19:47:00.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:00 smithi100 bash[80199]: cluster 2024-07-28T19:47:00.228656+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-07-28T19:47:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:00 smithi100 bash[82644]: cluster 2024-07-28T19:46:59.395439+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-07-28T19:47:00.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:00 smithi100 bash[82644]: cluster 2024-07-28T19:47:00.228656+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-07-28T19:47:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:00 smithi138 bash[67894]: cluster 2024-07-28T19:46:59.395439+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-07-28T19:47:00.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:00 smithi138 bash[67894]: cluster 2024-07-28T19:47:00.228656+0000 mon.a (mon.0) 1346 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-07-28T19:47:01.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:01 smithi100 bash[80199]: cluster 2024-07-28T19:47:00.014140+0000 mgr.y (mgr.25358) 432 : cluster [DBG] pgmap v868: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 45 op/s 2024-07-28T19:47:01.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:01 smithi100 bash[80199]: cluster 2024-07-28T19:47:01.235060+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-07-28T19:47:01.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:01 smithi100 bash[82644]: cluster 2024-07-28T19:47:00.014140+0000 mgr.y (mgr.25358) 432 : cluster [DBG] pgmap v868: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 45 op/s 2024-07-28T19:47:01.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:01 smithi100 bash[82644]: cluster 2024-07-28T19:47:01.235060+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-07-28T19:47:01.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:01 smithi138 bash[67894]: cluster 2024-07-28T19:47:00.014140+0000 mgr.y (mgr.25358) 432 : cluster [DBG] pgmap v868: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 45 op/s 2024-07-28T19:47:01.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:01 smithi138 bash[67894]: cluster 2024-07-28T19:47:01.235060+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-07-28T19:47:02.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:47:02 smithi138 bash[60615]: ts=2024-07-28T19:47:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:47:03.230 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:47:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:47:02] "GET /metrics HTTP/1.1" 200 82354 "" "Prometheus/2.43.0" 2024-07-28T19:47:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:03 smithi100 bash[80199]: cluster 2024-07-28T19:47:02.015306+0000 mgr.y (mgr.25358) 433 : cluster [DBG] pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 45 op/s 2024-07-28T19:47:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:03 smithi100 bash[80199]: cluster 2024-07-28T19:47:02.235966+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-07-28T19:47:03.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:03 smithi100 bash[82644]: cluster 2024-07-28T19:47:02.015306+0000 mgr.y (mgr.25358) 433 : cluster [DBG] pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 45 op/s 2024-07-28T19:47:03.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:03 smithi100 bash[82644]: cluster 2024-07-28T19:47:02.235966+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-07-28T19:47:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:03 smithi138 bash[67894]: cluster 2024-07-28T19:47:02.015306+0000 mgr.y (mgr.25358) 433 : cluster [DBG] pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 45 op/s 2024-07-28T19:47:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:03 smithi138 bash[67894]: cluster 2024-07-28T19:47:02.235966+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-07-28T19:47:04.495 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] TestLibRBD.ConcurentOperations (25911 ms) 2024-07-28T19:47:04.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:04 smithi138 bash[67894]: cluster 2024-07-28T19:47:03.247273+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-07-28T19:47:04.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:04 smithi100 bash[80199]: cluster 2024-07-28T19:47:03.247273+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-07-28T19:47:04.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:04 smithi100 bash[82644]: cluster 2024-07-28T19:47:03.247273+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-07-28T19:47:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:05 smithi138 bash[67894]: cluster 2024-07-28T19:47:04.018835+0000 mgr.y (mgr.25358) 434 : cluster [DBG] pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 78 op/s 2024-07-28T19:47:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:05 smithi138 bash[67894]: cluster 2024-07-28T19:47:04.260736+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-07-28T19:47:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:05 smithi138 bash[67894]: cluster 2024-07-28T19:47:05.233451+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-07-28T19:47:05.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:05 smithi100 bash[80199]: cluster 2024-07-28T19:47:04.018835+0000 mgr.y (mgr.25358) 434 : cluster [DBG] pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 78 op/s 2024-07-28T19:47:05.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:05 smithi100 bash[80199]: cluster 2024-07-28T19:47:04.260736+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-07-28T19:47:05.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:05 smithi100 bash[80199]: cluster 2024-07-28T19:47:05.233451+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-07-28T19:47:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:05 smithi100 bash[82644]: cluster 2024-07-28T19:47:04.018835+0000 mgr.y (mgr.25358) 434 : cluster [DBG] pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 4.5 KiB/s wr, 78 op/s 2024-07-28T19:47:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:05 smithi100 bash[82644]: cluster 2024-07-28T19:47:04.260736+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-07-28T19:47:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:05 smithi100 bash[82644]: cluster 2024-07-28T19:47:05.233451+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-07-28T19:47:07.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:07 smithi100 bash[80199]: cluster 2024-07-28T19:47:06.019986+0000 mgr.y (mgr.25358) 435 : cluster [DBG] pgmap v877: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 41 op/s 2024-07-28T19:47:07.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:07 smithi100 bash[80199]: cluster 2024-07-28T19:47:06.234454+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-07-28T19:47:07.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:07 smithi100 bash[82644]: cluster 2024-07-28T19:47:06.019986+0000 mgr.y (mgr.25358) 435 : cluster [DBG] pgmap v877: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 41 op/s 2024-07-28T19:47:07.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:07 smithi100 bash[82644]: cluster 2024-07-28T19:47:06.234454+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-07-28T19:47:07.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:07 smithi138 bash[67894]: cluster 2024-07-28T19:47:06.019986+0000 mgr.y (mgr.25358) 435 : cluster [DBG] pgmap v877: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 41 op/s 2024-07-28T19:47:07.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:07 smithi138 bash[67894]: cluster 2024-07-28T19:47:06.234454+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-07-28T19:47:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:08 smithi138 bash[67894]: cluster 2024-07-28T19:47:07.239344+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-07-28T19:47:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:08 smithi138 bash[67894]: audit 2024-07-28T19:47:07.641794+0000 mon.a (mon.0) 1354 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:08 smithi138 bash[67894]: cluster 2024-07-28T19:47:08.242662+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-07-28T19:47:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:08 smithi100 bash[80199]: cluster 2024-07-28T19:47:07.239344+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-07-28T19:47:08.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:08 smithi100 bash[80199]: audit 2024-07-28T19:47:07.641794+0000 mon.a (mon.0) 1354 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:08.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:08 smithi100 bash[80199]: cluster 2024-07-28T19:47:08.242662+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-07-28T19:47:08.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:08 smithi100 bash[82644]: cluster 2024-07-28T19:47:07.239344+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-07-28T19:47:08.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:08 smithi100 bash[82644]: audit 2024-07-28T19:47:07.641794+0000 mon.a (mon.0) 1354 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:08.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:08 smithi100 bash[82644]: cluster 2024-07-28T19:47:08.242662+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-07-28T19:47:09.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:09 smithi138 bash[67894]: cluster 2024-07-28T19:47:08.021896+0000 mgr.y (mgr.25358) 436 : cluster [DBG] pgmap v880: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:09.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:09 smithi138 bash[67894]: cluster 2024-07-28T19:47:09.243553+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-07-28T19:47:09.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:09 smithi100 bash[80199]: cluster 2024-07-28T19:47:08.021896+0000 mgr.y (mgr.25358) 436 : cluster [DBG] pgmap v880: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:09.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:09 smithi100 bash[80199]: cluster 2024-07-28T19:47:09.243553+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-07-28T19:47:09.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:09 smithi100 bash[82644]: cluster 2024-07-28T19:47:08.021896+0000 mgr.y (mgr.25358) 436 : cluster [DBG] pgmap v880: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:09.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:09 smithi100 bash[82644]: cluster 2024-07-28T19:47:09.243553+0000 mon.a (mon.0) 1356 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-07-28T19:47:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:11 smithi100 bash[80199]: cluster 2024-07-28T19:47:10.023488+0000 mgr.y (mgr.25358) 437 : cluster [DBG] pgmap v883: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:11.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:11 smithi100 bash[80199]: cluster 2024-07-28T19:47:10.239175+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-07-28T19:47:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:11 smithi100 bash[82644]: cluster 2024-07-28T19:47:10.023488+0000 mgr.y (mgr.25358) 437 : cluster [DBG] pgmap v883: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:11 smithi100 bash[82644]: cluster 2024-07-28T19:47:10.239175+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-07-28T19:47:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:11 smithi138 bash[67894]: cluster 2024-07-28T19:47:10.023488+0000 mgr.y (mgr.25358) 437 : cluster [DBG] pgmap v883: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:11 smithi138 bash[67894]: cluster 2024-07-28T19:47:10.239175+0000 mon.a (mon.0) 1357 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-07-28T19:47:12.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:47:12 smithi138 bash[60615]: ts=2024-07-28T19:47:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:47:12.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:12 smithi138 bash[67894]: cluster 2024-07-28T19:47:11.238076+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-07-28T19:47:12.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:12 smithi100 bash[80199]: cluster 2024-07-28T19:47:11.238076+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-07-28T19:47:12.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:12 smithi100 bash[82644]: cluster 2024-07-28T19:47:11.238076+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-07-28T19:47:13.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:47:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:47:12] "GET /metrics HTTP/1.1" 200 75136 "" "Prometheus/2.43.0" 2024-07-28T19:47:13.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:13 smithi138 bash[67894]: cluster 2024-07-28T19:47:12.024470+0000 mgr.y (mgr.25358) 438 : cluster [DBG] pgmap v886: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:13.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:13 smithi138 bash[67894]: cluster 2024-07-28T19:47:12.243170+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-07-28T19:47:13.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:13 smithi138 bash[67894]: cluster 2024-07-28T19:47:13.243831+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-07-28T19:47:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:13 smithi100 bash[80199]: cluster 2024-07-28T19:47:12.024470+0000 mgr.y (mgr.25358) 438 : cluster [DBG] pgmap v886: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:13.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:13 smithi100 bash[80199]: cluster 2024-07-28T19:47:12.243170+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-07-28T19:47:13.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:13 smithi100 bash[80199]: cluster 2024-07-28T19:47:13.243831+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-07-28T19:47:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:13 smithi100 bash[82644]: cluster 2024-07-28T19:47:12.024470+0000 mgr.y (mgr.25358) 438 : cluster [DBG] pgmap v886: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:13 smithi100 bash[82644]: cluster 2024-07-28T19:47:12.243170+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-07-28T19:47:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:13 smithi100 bash[82644]: cluster 2024-07-28T19:47:13.243831+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-07-28T19:47:15.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:15 smithi100 bash[82644]: cluster 2024-07-28T19:47:14.026732+0000 mgr.y (mgr.25358) 439 : cluster [DBG] pgmap v889: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:15.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:15 smithi100 bash[82644]: cluster 2024-07-28T19:47:14.246591+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-07-28T19:47:15.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:15 smithi100 bash[82644]: cluster 2024-07-28T19:47:15.238030+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-07-28T19:47:15.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:15 smithi100 bash[80199]: cluster 2024-07-28T19:47:14.026732+0000 mgr.y (mgr.25358) 439 : cluster [DBG] pgmap v889: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:15.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:15 smithi100 bash[80199]: cluster 2024-07-28T19:47:14.246591+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-07-28T19:47:15.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:15 smithi100 bash[80199]: cluster 2024-07-28T19:47:15.238030+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-07-28T19:47:15.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:15 smithi138 bash[67894]: cluster 2024-07-28T19:47:14.026732+0000 mgr.y (mgr.25358) 439 : cluster [DBG] pgmap v889: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:15.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:15 smithi138 bash[67894]: cluster 2024-07-28T19:47:14.246591+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-07-28T19:47:15.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:15 smithi138 bash[67894]: cluster 2024-07-28T19:47:15.238030+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-07-28T19:47:17.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:17 smithi100 bash[82644]: cluster 2024-07-28T19:47:16.027605+0000 mgr.y (mgr.25358) 440 : cluster [DBG] pgmap v892: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:17.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:17 smithi100 bash[82644]: cluster 2024-07-28T19:47:16.239500+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-07-28T19:47:17.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:17 smithi100 bash[80199]: cluster 2024-07-28T19:47:16.027605+0000 mgr.y (mgr.25358) 440 : cluster [DBG] pgmap v892: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:17.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:17 smithi100 bash[80199]: cluster 2024-07-28T19:47:16.239500+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-07-28T19:47:17.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:17 smithi138 bash[67894]: cluster 2024-07-28T19:47:16.027605+0000 mgr.y (mgr.25358) 440 : cluster [DBG] pgmap v892: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:17.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:17 smithi138 bash[67894]: cluster 2024-07-28T19:47:16.239500+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-07-28T19:47:18.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:18 smithi138 bash[67894]: cluster 2024-07-28T19:47:17.242556+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-07-28T19:47:18.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:18 smithi138 bash[67894]: cluster 2024-07-28T19:47:18.246628+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-07-28T19:47:18.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:18 smithi100 bash[80199]: cluster 2024-07-28T19:47:17.242556+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-07-28T19:47:18.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:18 smithi100 bash[80199]: cluster 2024-07-28T19:47:18.246628+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-07-28T19:47:18.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:18 smithi100 bash[82644]: cluster 2024-07-28T19:47:17.242556+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-07-28T19:47:18.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:18 smithi100 bash[82644]: cluster 2024-07-28T19:47:18.246628+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-07-28T19:47:19.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:19 smithi138 bash[67894]: cluster 2024-07-28T19:47:18.028807+0000 mgr.y (mgr.25358) 441 : cluster [DBG] pgmap v895: 297 pgs: 297 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:19.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:19 smithi138 bash[67894]: cluster 2024-07-28T19:47:19.248884+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-07-28T19:47:19.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:19 smithi100 bash[80199]: cluster 2024-07-28T19:47:18.028807+0000 mgr.y (mgr.25358) 441 : cluster [DBG] pgmap v895: 297 pgs: 297 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:19.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:19 smithi100 bash[80199]: cluster 2024-07-28T19:47:19.248884+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-07-28T19:47:19.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:19 smithi100 bash[82644]: cluster 2024-07-28T19:47:18.028807+0000 mgr.y (mgr.25358) 441 : cluster [DBG] pgmap v895: 297 pgs: 297 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:19.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:19 smithi100 bash[82644]: cluster 2024-07-28T19:47:19.248884+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-07-28T19:47:21.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:21 smithi138 bash[67894]: cluster 2024-07-28T19:47:20.029880+0000 mgr.y (mgr.25358) 442 : cluster [DBG] pgmap v898: 233 pgs: 233 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:21.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:21 smithi138 bash[67894]: cluster 2024-07-28T19:47:20.250920+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-07-28T19:47:21.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:21 smithi100 bash[80199]: cluster 2024-07-28T19:47:20.029880+0000 mgr.y (mgr.25358) 442 : cluster [DBG] pgmap v898: 233 pgs: 233 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:21.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:21 smithi100 bash[80199]: cluster 2024-07-28T19:47:20.250920+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-07-28T19:47:21.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:21 smithi100 bash[82644]: cluster 2024-07-28T19:47:20.029880+0000 mgr.y (mgr.25358) 442 : cluster [DBG] pgmap v898: 233 pgs: 233 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:21.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:21 smithi100 bash[82644]: cluster 2024-07-28T19:47:20.250920+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-07-28T19:47:22.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:22 smithi138 bash[67894]: cluster 2024-07-28T19:47:21.254160+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-07-28T19:47:22.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:22 smithi138 bash[67894]: cluster 2024-07-28T19:47:22.256437+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-07-28T19:47:22.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:47:22 smithi138 bash[60615]: ts=2024-07-28T19:47:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:47:22.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:22 smithi100 bash[82644]: cluster 2024-07-28T19:47:21.254160+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-07-28T19:47:22.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:22 smithi100 bash[82644]: cluster 2024-07-28T19:47:22.256437+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-07-28T19:47:22.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:22 smithi100 bash[80199]: cluster 2024-07-28T19:47:21.254160+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-07-28T19:47:22.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:22 smithi100 bash[80199]: cluster 2024-07-28T19:47:22.256437+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-07-28T19:47:23.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:47:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:47:22] "GET /metrics HTTP/1.1" 200 39079 "" "Prometheus/2.43.0" 2024-07-28T19:47:23.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:23 smithi138 bash[67894]: cluster 2024-07-28T19:47:22.030582+0000 mgr.y (mgr.25358) 443 : cluster [DBG] pgmap v901: 169 pgs: 169 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:23.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:23 smithi138 bash[67894]: audit 2024-07-28T19:47:22.641745+0000 mon.a (mon.0) 1370 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:23.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:23 smithi138 bash[67894]: cluster 2024-07-28T19:47:23.259132+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-07-28T19:47:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:23 smithi100 bash[80199]: cluster 2024-07-28T19:47:22.030582+0000 mgr.y (mgr.25358) 443 : cluster [DBG] pgmap v901: 169 pgs: 169 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:23 smithi100 bash[80199]: audit 2024-07-28T19:47:22.641745+0000 mon.a (mon.0) 1370 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:23 smithi100 bash[80199]: cluster 2024-07-28T19:47:23.259132+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-07-28T19:47:23.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:23 smithi100 bash[82644]: cluster 2024-07-28T19:47:22.030582+0000 mgr.y (mgr.25358) 443 : cluster [DBG] pgmap v901: 169 pgs: 169 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:23.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:23 smithi100 bash[82644]: audit 2024-07-28T19:47:22.641745+0000 mon.a (mon.0) 1370 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:23.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:23 smithi100 bash[82644]: cluster 2024-07-28T19:47:23.259132+0000 mon.a (mon.0) 1371 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-07-28T19:47:24.267 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 160 tests from TestLibRBD (383301 ms total) 2024-07-28T19:47:24.267 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:47:24.267 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 11 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-07-28T19:47:24.321 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-07-28T19:47:25.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:25 smithi138 bash[67894]: cluster 2024-07-28T19:47:24.031541+0000 mgr.y (mgr.25358) 444 : cluster [DBG] pgmap v904: 105 pgs: 105 active+clean; 586 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:25.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:25 smithi138 bash[67894]: cluster 2024-07-28T19:47:24.262616+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-07-28T19:47:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:25 smithi100 bash[80199]: cluster 2024-07-28T19:47:24.031541+0000 mgr.y (mgr.25358) 444 : cluster [DBG] pgmap v904: 105 pgs: 105 active+clean; 586 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:25.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:25 smithi100 bash[80199]: cluster 2024-07-28T19:47:24.262616+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-07-28T19:47:25.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:25 smithi100 bash[82644]: cluster 2024-07-28T19:47:24.031541+0000 mgr.y (mgr.25358) 444 : cluster [DBG] pgmap v904: 105 pgs: 105 active+clean; 586 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:25.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:25 smithi100 bash[82644]: cluster 2024-07-28T19:47:24.262616+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-07-28T19:47:26.309 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:47:26.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:26 smithi138 bash[67894]: cluster 2024-07-28T19:47:25.291064+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-07-28T19:47:26.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:26 smithi138 bash[67894]: audit 2024-07-28T19:47:25.300839+0000 mon.a (mon.0) 1374 : audit [INF] from='client.? 172.21.15.100:0/3273371884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:47:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:26 smithi100 bash[80199]: cluster 2024-07-28T19:47:25.291064+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-07-28T19:47:26.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:26 smithi100 bash[80199]: audit 2024-07-28T19:47:25.300839+0000 mon.a (mon.0) 1374 : audit [INF] from='client.? 172.21.15.100:0/3273371884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:47:26.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:26 smithi100 bash[82644]: cluster 2024-07-28T19:47:25.291064+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-07-28T19:47:26.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:26 smithi100 bash[82644]: audit 2024-07-28T19:47:25.300839+0000 mon.a (mon.0) 1374 : audit [INF] from='client.? 172.21.15.100:0/3273371884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:47:27.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:27 smithi100 bash[80199]: cluster 2024-07-28T19:47:26.032239+0000 mgr.y (mgr.25358) 445 : cluster [DBG] pgmap v907: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:27.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:27 smithi100 bash[80199]: cluster 2024-07-28T19:47:26.283985+0000 mon.a (mon.0) 1375 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:47:27.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:27 smithi100 bash[80199]: audit 2024-07-28T19:47:26.300044+0000 mon.a (mon.0) 1376 : audit [INF] from='client.? 172.21.15.100:0/3273371884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:47:27.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:27 smithi100 bash[80199]: cluster 2024-07-28T19:47:26.300567+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-07-28T19:47:27.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:27 smithi100 bash[82644]: cluster 2024-07-28T19:47:26.032239+0000 mgr.y (mgr.25358) 445 : cluster [DBG] pgmap v907: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:27.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:27 smithi100 bash[82644]: cluster 2024-07-28T19:47:26.283985+0000 mon.a (mon.0) 1375 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:47:27.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:27 smithi100 bash[82644]: audit 2024-07-28T19:47:26.300044+0000 mon.a (mon.0) 1376 : audit [INF] from='client.? 172.21.15.100:0/3273371884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:47:27.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:27 smithi100 bash[82644]: cluster 2024-07-28T19:47:26.300567+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-07-28T19:47:27.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:27 smithi138 bash[67894]: cluster 2024-07-28T19:47:26.032239+0000 mgr.y (mgr.25358) 445 : cluster [DBG] pgmap v907: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:27.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:27 smithi138 bash[67894]: cluster 2024-07-28T19:47:26.283985+0000 mon.a (mon.0) 1375 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-07-28T19:47:27.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:27 smithi138 bash[67894]: audit 2024-07-28T19:47:26.300044+0000 mon.a (mon.0) 1376 : audit [INF] from='client.? 172.21.15.100:0/3273371884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:47:27.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:27 smithi138 bash[67894]: cluster 2024-07-28T19:47:26.300567+0000 mon.a (mon.0) 1377 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-07-28T19:47:28.380 INFO:tasks.workunit.client.0.smithi100.stdout: wrote [1667525~61107,1769892~100818,6477461~89924,8967495~4873,8982516~46700,10624378~11642,12393979~18562,15359068~14337,15486131~56955,20175547~8045] 2024-07-28T19:47:28.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:28 smithi100 bash[80199]: cluster 2024-07-28T19:47:27.304969+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-07-28T19:47:28.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:28 smithi100 bash[82644]: cluster 2024-07-28T19:47:27.304969+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-07-28T19:47:28.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:28 smithi138 bash[67894]: cluster 2024-07-28T19:47:27.304969+0000 mon.a (mon.0) 1378 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-07-28T19:47:29.363 INFO:tasks.workunit.client.0.smithi100.stdout: wrote [1261880~651,2466520~89427,3454867~3270,4506069~24516,8010985~41915,9930461~67943,10431182~21810,12443295~97839,13080185~18464,18547142~87912] 2024-07-28T19:47:29.379 INFO:tasks.workunit.client.0.smithi100.stdout: diff was [1261880~651,1870710~1587427,4506069~24516,6567385~1485515,9930461~67943,10431182~21810,12412541~128593,13080185~18464,18547142~87912] 2024-07-28T19:47:29.387 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5067 ms) 2024-07-28T19:47:29.387 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-07-28T19:47:29.388 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:47:29.443 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~256 2024-07-28T19:47:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:29 smithi100 bash[80199]: cluster 2024-07-28T19:47:28.033061+0000 mgr.y (mgr.25358) 446 : cluster [DBG] pgmap v910: 105 pgs: 24 unknown, 81 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:29.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:29 smithi100 bash[80199]: cluster 2024-07-28T19:47:28.310661+0000 mon.a (mon.0) 1379 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:47:29.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:29 smithi100 bash[80199]: cluster 2024-07-28T19:47:28.327453+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-07-28T19:47:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:29 smithi100 bash[82644]: cluster 2024-07-28T19:47:28.033061+0000 mgr.y (mgr.25358) 446 : cluster [DBG] pgmap v910: 105 pgs: 24 unknown, 81 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:29 smithi100 bash[82644]: cluster 2024-07-28T19:47:28.310661+0000 mon.a (mon.0) 1379 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:47:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:29 smithi100 bash[82644]: cluster 2024-07-28T19:47:28.327453+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-07-28T19:47:29.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:29 smithi138 bash[67894]: cluster 2024-07-28T19:47:28.033061+0000 mgr.y (mgr.25358) 446 : cluster [DBG] pgmap v910: 105 pgs: 24 unknown, 81 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:47:29.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:29 smithi138 bash[67894]: cluster 2024-07-28T19:47:28.310661+0000 mon.a (mon.0) 1379 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-07-28T19:47:29.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:29 smithi138 bash[67894]: cluster 2024-07-28T19:47:28.327453+0000 mon.a (mon.0) 1380 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-07-28T19:47:30.384 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~256 2024-07-28T19:47:30.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:30 smithi100 bash[80199]: cluster 2024-07-28T19:47:29.330331+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-07-28T19:47:30.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:30 smithi100 bash[82644]: cluster 2024-07-28T19:47:29.330331+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-07-28T19:47:30.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:30 smithi138 bash[67894]: cluster 2024-07-28T19:47:29.330331+0000 mon.a (mon.0) 1381 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-07-28T19:47:31.396 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~256 2024-07-28T19:47:31.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[80199]: cluster 2024-07-28T19:47:30.033813+0000 mgr.y (mgr.25358) 447 : cluster [DBG] pgmap v913: 105 pgs: 4 unknown, 101 active+clean; 3.4 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 21 KiB/s wr, 13 op/s 2024-07-28T19:47:31.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[80199]: cluster 2024-07-28T19:47:30.354514+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-07-28T19:47:31.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[80199]: audit 2024-07-28T19:47:30.632128+0000 mon.a (mon.0) 1383 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:47:31.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[80199]: audit 2024-07-28T19:47:30.999756+0000 mon.a (mon.0) 1384 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:31.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[80199]: audit 2024-07-28T19:47:31.004506+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:31.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[82644]: cluster 2024-07-28T19:47:30.033813+0000 mgr.y (mgr.25358) 447 : cluster [DBG] pgmap v913: 105 pgs: 4 unknown, 101 active+clean; 3.4 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 21 KiB/s wr, 13 op/s 2024-07-28T19:47:31.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[82644]: cluster 2024-07-28T19:47:30.354514+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-07-28T19:47:31.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[82644]: audit 2024-07-28T19:47:30.632128+0000 mon.a (mon.0) 1383 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:47:31.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[82644]: audit 2024-07-28T19:47:30.999756+0000 mon.a (mon.0) 1384 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:31.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:31 smithi100 bash[82644]: audit 2024-07-28T19:47:31.004506+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:31.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:31 smithi138 bash[67894]: cluster 2024-07-28T19:47:30.033813+0000 mgr.y (mgr.25358) 447 : cluster [DBG] pgmap v913: 105 pgs: 4 unknown, 101 active+clean; 3.4 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 21 KiB/s wr, 13 op/s 2024-07-28T19:47:31.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:31 smithi138 bash[67894]: cluster 2024-07-28T19:47:30.354514+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-07-28T19:47:31.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:31 smithi138 bash[67894]: audit 2024-07-28T19:47:30.632128+0000 mon.a (mon.0) 1383 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:47:31.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:31 smithi138 bash[67894]: audit 2024-07-28T19:47:30.999756+0000 mon.a (mon.0) 1384 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:31.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:31 smithi138 bash[67894]: audit 2024-07-28T19:47:31.004506+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:32.402 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~256 2024-07-28T19:47:32.405 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (3017 ms) 2024-07-28T19:47:32.405 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-07-28T19:47:32.405 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:47:32.533 INFO:tasks.workunit.client.0.smithi100.stdout: i=0 exists [190310018~7476016,220192808~8161493,270155517~1823588,304931669~1863812,315266456~4252661,341344605~3505462,353711774~4789952,386565670~6473058,403083722~898212,405615454~1667126] wrote [190310018~7476016,220192808~8161493,270155517~1823588,304931669~1863812,315266456~4252661,341344605~3505462,353711774~4789952,386565670~6473058,403083722~898212,405615454~1667126] 2024-07-28T19:47:32.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:32 smithi100 bash[80199]: cluster 2024-07-28T19:47:31.352485+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-07-28T19:47:32.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:32 smithi100 bash[82644]: cluster 2024-07-28T19:47:31.352485+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-07-28T19:47:32.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:32 smithi138 bash[67894]: cluster 2024-07-28T19:47:31.352485+0000 mon.a (mon.0) 1386 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-07-28T19:47:32.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:47:32 smithi138 bash[60615]: ts=2024-07-28T19:47:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:47:33.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:47:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:47:32] "GET /metrics HTTP/1.1" 200 39079 "" "Prometheus/2.43.0" 2024-07-28T19:47:33.498 INFO:tasks.workunit.client.0.smithi100.stdout: i=1 exists [11015273~2290920,32983006~6923046,58061600~4881219,66565039~8114072,141880579~2515951,147636173~6380355,190310018~7476016,220192808~8161493,241226272~5376814,270155517~1823588,304931669~1863812,315266456~4252661,328798819~3346608,341344605~3505462,353711774~4789952,366220121~5452321,386565670~6473058,403083722~898212,405615454~1667126] wrote [11015273~2290920,32983006~6923046,58061600~4881219,66565039~8114072,141880579~2515951,147636173~6380355,241226272~5376814,328798819~3346608,366220121~5452321] 2024-07-28T19:47:33.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:33 smithi100 bash[80199]: cluster 2024-07-28T19:47:32.034481+0000 mgr.y (mgr.25358) 448 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 123 KiB/s wr, 16 op/s 2024-07-28T19:47:33.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:33 smithi100 bash[80199]: cluster 2024-07-28T19:47:32.372922+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-07-28T19:47:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:33 smithi100 bash[82644]: cluster 2024-07-28T19:47:32.034481+0000 mgr.y (mgr.25358) 448 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 123 KiB/s wr, 16 op/s 2024-07-28T19:47:33.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:33 smithi100 bash[82644]: cluster 2024-07-28T19:47:32.372922+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-07-28T19:47:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:33 smithi138 bash[67894]: cluster 2024-07-28T19:47:32.034481+0000 mgr.y (mgr.25358) 448 : cluster [DBG] pgmap v916: 105 pgs: 105 active+clean; 10 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 123 KiB/s wr, 16 op/s 2024-07-28T19:47:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:33 smithi138 bash[67894]: cluster 2024-07-28T19:47:32.372922+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-07-28T19:47:34.523 INFO:tasks.workunit.client.0.smithi100.stdout: i=2 exists [11015273~2290920,26761270~3620690,32983006~6923046,58061600~4881219,66565039~8114072,141880579~2515951,147636173~6380355,161173157~3951100,190310018~7476016,220192808~8161493,229242050~1098587,236800222~9802864,270155517~1823588,303607713~4909202,315266456~4252661,328798819~3346608,341344605~3505462,351158777~7342949,366220121~5452321,386565670~6473058,403083722~898212,405615454~1667126,409516070~3951114] wrote [26761270~3620690,68867624~4203403,161173157~3951100,229242050~1098587,236800222~4533966,303607713~4909202,351158777~5921015,409516070~3951114] 2024-07-28T19:47:34.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:34 smithi100 bash[80199]: cluster 2024-07-28T19:47:33.382895+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-07-28T19:47:34.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:34 smithi100 bash[82644]: cluster 2024-07-28T19:47:33.382895+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-07-28T19:47:34.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:34 smithi138 bash[67894]: cluster 2024-07-28T19:47:33.382895+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-07-28T19:47:35.353 INFO:tasks.workunit.client.0.smithi100.stdout: i=3 exists [11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,58061600~4881219,66565039~8114072,77096290~1863886,141880579~2515951,147636173~6380355,161173157~3951100,167752029~510497,190310018~7476016,220192808~8161493,229242050~1098587,236800222~9802864,270155517~1823588,281629645~5260608,288011974~6470403,299052751~9464164,315266456~4252661,326954272~5191155,341344605~6887593,351158777~7342949,366220121~5452321,386565670~6473058,403083722~898212,405615454~1667126,409516070~3951114] wrote [31967198~1848860,49157145~3421699,77096290~1863886,167752029~510497,281629645~5260608,288011974~6470403,299052751~6022892,326954272~2883588,343359455~4872743] 2024-07-28T19:47:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:35 smithi100 bash[80199]: cluster 2024-07-28T19:47:34.035464+0000 mgr.y (mgr.25358) 449 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 48 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 4.7 MiB/s wr, 154 op/s 2024-07-28T19:47:35.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:35 smithi100 bash[80199]: cluster 2024-07-28T19:47:34.397817+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-07-28T19:47:35.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:35 smithi100 bash[80199]: cluster 2024-07-28T19:47:35.260081+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-07-28T19:47:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:35 smithi100 bash[82644]: cluster 2024-07-28T19:47:34.035464+0000 mgr.y (mgr.25358) 449 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 48 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 4.7 MiB/s wr, 154 op/s 2024-07-28T19:47:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:35 smithi100 bash[82644]: cluster 2024-07-28T19:47:34.397817+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-07-28T19:47:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:35 smithi100 bash[82644]: cluster 2024-07-28T19:47:35.260081+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-07-28T19:47:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:35 smithi138 bash[67894]: cluster 2024-07-28T19:47:34.035464+0000 mgr.y (mgr.25358) 449 : cluster [DBG] pgmap v919: 105 pgs: 105 active+clean; 48 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 4.7 MiB/s wr, 154 op/s 2024-07-28T19:47:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:35 smithi138 bash[67894]: cluster 2024-07-28T19:47:34.397817+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-07-28T19:47:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:35 smithi138 bash[67894]: cluster 2024-07-28T19:47:35.260081+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-07-28T19:47:36.483 INFO:tasks.workunit.client.0.smithi100.stdout: i=4 exists [11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,58061600~4881219,66565039~8114072,77096290~1863886,141880579~2515951,147636173~6380355,161173157~3951100,167752029~510497,188958486~8827548,201778320~5844812,216071836~12282465,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,270155517~2642034,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,315266456~4252661,326954272~5191155,337654823~1447438,341344605~6887593,351158777~7342949,366220121~5452321,381707621~11331107,403083722~898212,405615454~1667126,409516070~3951114] wrote [188958486~1816264,201778320~5844812,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,337654823~1447438,367354362~1092722,381707621~5380293] 2024-07-28T19:47:37.621 INFO:tasks.workunit.client.0.smithi100.stdout: i=5 exists [112226~7216589,11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,188958486~8827548,201778320~5844812,209422996~4801946,216071836~12282465,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,270155517~2642034,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,315266456~4252661,326954272~5191155,337654823~1447438,341344605~6887593,351158777~7342949,366220121~5452321,381707621~11331107,395829986~13298434,409516070~3951114] wrote [112226~7216589,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,152843223~4376530,209422996~4801946,395829986~13298434] 2024-07-28T19:47:37.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: cluster 2024-07-28T19:47:36.035882+0000 mgr.y (mgr.25358) 450 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 97 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 104 KiB/s rd, 13 MiB/s wr, 173 op/s 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: cluster 2024-07-28T19:47:36.416208+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: audit 2024-07-28T19:47:36.502183+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: audit 2024-07-28T19:47:36.520039+0000 mon.a (mon.0) 1393 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: audit 2024-07-28T19:47:36.559329+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: audit 2024-07-28T19:47:36.564831+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: audit 2024-07-28T19:47:37.023733+0000 mon.a (mon.0) 1396 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: audit 2024-07-28T19:47:37.024982+0000 mon.a (mon.0) 1397 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[80199]: audit 2024-07-28T19:47:37.033935+0000 mon.a (mon.0) 1398 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: cluster 2024-07-28T19:47:36.035882+0000 mgr.y (mgr.25358) 450 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 97 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 104 KiB/s rd, 13 MiB/s wr, 173 op/s 2024-07-28T19:47:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: cluster 2024-07-28T19:47:36.416208+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-07-28T19:47:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: audit 2024-07-28T19:47:36.502183+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: audit 2024-07-28T19:47:36.520039+0000 mon.a (mon.0) 1393 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: audit 2024-07-28T19:47:36.559329+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: audit 2024-07-28T19:47:36.564831+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: audit 2024-07-28T19:47:37.023733+0000 mon.a (mon.0) 1396 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:47:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: audit 2024-07-28T19:47:37.024982+0000 mon.a (mon.0) 1397 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:47:37.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:37 smithi100 bash[82644]: audit 2024-07-28T19:47:37.033935+0000 mon.a (mon.0) 1398 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: cluster 2024-07-28T19:47:36.035882+0000 mgr.y (mgr.25358) 450 : cluster [DBG] pgmap v922: 105 pgs: 105 active+clean; 97 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 104 KiB/s rd, 13 MiB/s wr, 173 op/s 2024-07-28T19:47:37.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: cluster 2024-07-28T19:47:36.416208+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-07-28T19:47:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: audit 2024-07-28T19:47:36.502183+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: audit 2024-07-28T19:47:36.520039+0000 mon.a (mon.0) 1393 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: audit 2024-07-28T19:47:36.559329+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: audit 2024-07-28T19:47:36.564831+0000 mon.a (mon.0) 1395 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: audit 2024-07-28T19:47:37.023733+0000 mon.a (mon.0) 1396 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:47:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: audit 2024-07-28T19:47:37.024982+0000 mon.a (mon.0) 1397 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:47:37.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:37 smithi138 bash[67894]: audit 2024-07-28T19:47:37.033935+0000 mon.a (mon.0) 1398 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:38.631 INFO:tasks.workunit.client.0.smithi100.stdout: i=6 exists [112226~13193967,16897474~3285069,26761270~3620690,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,188958486~8827548,201778320~5844812,209422996~4801946,216071836~12282465,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,264356515~4168024,270155517~2642034,276135342~3924938,281629645~5260608,287356099~7126278,299052751~9464164,315266456~4252661,320568422~11577005,337654823~1447438,341344605~6887593,351158777~7342949,366220121~5452321,381707621~11331107,395829986~13298434,409516070~3951114] wrote [7209425~6089872,16897474~3285069,117774156~2460779,153891366~718136,220349894~4833028,264356515~4168024,282017125~1092925,287356099~1845842,320568422~7006352,410737048~1385636] 2024-07-28T19:47:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:38 smithi138 bash[67894]: cluster 2024-07-28T19:47:37.527489+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-07-28T19:47:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:38 smithi138 bash[67894]: audit 2024-07-28T19:47:37.643303+0000 mon.a (mon.0) 1400 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:38 smithi138 bash[67894]: cluster 2024-07-28T19:47:38.036633+0000 mgr.y (mgr.25358) 451 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 167 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 23 MiB/s wr, 91 op/s 2024-07-28T19:47:38.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:38 smithi100 bash[80199]: cluster 2024-07-28T19:47:37.527489+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-07-28T19:47:38.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:38 smithi100 bash[80199]: audit 2024-07-28T19:47:37.643303+0000 mon.a (mon.0) 1400 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:38.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:38 smithi100 bash[80199]: cluster 2024-07-28T19:47:38.036633+0000 mgr.y (mgr.25358) 451 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 167 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 23 MiB/s wr, 91 op/s 2024-07-28T19:47:38.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:38 smithi100 bash[82644]: cluster 2024-07-28T19:47:37.527489+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-07-28T19:47:38.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:38 smithi100 bash[82644]: audit 2024-07-28T19:47:37.643303+0000 mon.a (mon.0) 1400 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:38.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:38 smithi100 bash[82644]: cluster 2024-07-28T19:47:38.036633+0000 mgr.y (mgr.25358) 451 : cluster [DBG] pgmap v925: 105 pgs: 105 active+clean; 167 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 23 MiB/s wr, 91 op/s 2024-07-28T19:47:39.638 INFO:tasks.workunit.client.0.smithi100.stdout: i=7 exists [112226~13193967,16897474~3285069,26761270~3896241,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,126138082~3957397,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,174554202~4352940,188958486~8827548,201778320~5844812,209422996~4801946,216071836~12282465,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~10712203,320568422~11577005,337654823~1447438,341344605~6887593,351158777~7342949,366220121~5452321,381707621~11331107,395829986~13298434,409516070~3951114] wrote [27561189~3096322,65561507~2050765,126138082~3957397,174554202~4352940,225682819~1876237,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545] 2024-07-28T19:47:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:39 smithi138 bash[67894]: cluster 2024-07-28T19:47:38.539461+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-07-28T19:47:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:39 smithi138 bash[67894]: audit 2024-07-28T19:47:38.729771+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:39.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:39 smithi100 bash[80199]: cluster 2024-07-28T19:47:38.539461+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-07-28T19:47:39.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:39 smithi100 bash[80199]: audit 2024-07-28T19:47:38.729771+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:39.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:39 smithi100 bash[82644]: cluster 2024-07-28T19:47:38.539461+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-07-28T19:47:39.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:39 smithi100 bash[82644]: audit 2024-07-28T19:47:38.729771+0000 mon.a (mon.0) 1402 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:47:40.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:40 smithi138 bash[67894]: cluster 2024-07-28T19:47:39.538949+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-07-28T19:47:40.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:40 smithi138 bash[67894]: cluster 2024-07-28T19:47:40.037409+0000 mgr.y (mgr.25358) 452 : cluster [DBG] pgmap v928: 105 pgs: 105 active+clean; 288 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 159 KiB/s rd, 38 MiB/s wr, 261 op/s 2024-07-28T19:47:40.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:40 smithi138 bash[67894]: audit 2024-07-28T19:47:40.125330+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]: dispatch 2024-07-28T19:47:40.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:40 smithi138 bash[67894]: audit 2024-07-28T19:47:40.125799+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-07-28T19:47:40.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:40 smithi138 bash[67894]: audit 2024-07-28T19:47:40.126192+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 7]}]: dispatch 2024-07-28T19:47:40.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:40 smithi138 bash[67894]: audit 2024-07-28T19:47:40.126547+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]: dispatch 2024-07-28T19:47:40.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[80199]: cluster 2024-07-28T19:47:39.538949+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[80199]: cluster 2024-07-28T19:47:40.037409+0000 mgr.y (mgr.25358) 452 : cluster [DBG] pgmap v928: 105 pgs: 105 active+clean; 288 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 159 KiB/s rd, 38 MiB/s wr, 261 op/s 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[80199]: audit 2024-07-28T19:47:40.125330+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]: dispatch 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[80199]: audit 2024-07-28T19:47:40.125799+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[80199]: audit 2024-07-28T19:47:40.126192+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 7]}]: dispatch 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[80199]: audit 2024-07-28T19:47:40.126547+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]: dispatch 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[82644]: cluster 2024-07-28T19:47:39.538949+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[82644]: cluster 2024-07-28T19:47:40.037409+0000 mgr.y (mgr.25358) 452 : cluster [DBG] pgmap v928: 105 pgs: 105 active+clean; 288 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 159 KiB/s rd, 38 MiB/s wr, 261 op/s 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[82644]: audit 2024-07-28T19:47:40.125330+0000 mon.a (mon.0) 1404 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]: dispatch 2024-07-28T19:47:40.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[82644]: audit 2024-07-28T19:47:40.125799+0000 mon.a (mon.0) 1405 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-07-28T19:47:41.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[82644]: audit 2024-07-28T19:47:40.126192+0000 mon.a (mon.0) 1406 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 7]}]: dispatch 2024-07-28T19:47:41.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:40 smithi100 bash[82644]: audit 2024-07-28T19:47:40.126547+0000 mon.a (mon.0) 1407 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]: dispatch 2024-07-28T19:47:41.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[80199]: audit 2024-07-28T19:47:40.558410+0000 mon.a (mon.0) 1408 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[80199]: audit 2024-07-28T19:47:40.558541+0000 mon.a (mon.0) 1409 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[80199]: audit 2024-07-28T19:47:40.558706+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 7]}]': finished 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[80199]: audit 2024-07-28T19:47:40.558840+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]': finished 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[80199]: cluster 2024-07-28T19:47:40.559054+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[82644]: audit 2024-07-28T19:47:40.558410+0000 mon.a (mon.0) 1408 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[82644]: audit 2024-07-28T19:47:40.558541+0000 mon.a (mon.0) 1409 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[82644]: audit 2024-07-28T19:47:40.558706+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 7]}]': finished 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[82644]: audit 2024-07-28T19:47:40.558840+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]': finished 2024-07-28T19:47:41.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:41 smithi100 bash[82644]: cluster 2024-07-28T19:47:40.559054+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-07-28T19:47:42.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:41 smithi138 bash[67894]: audit 2024-07-28T19:47:40.558410+0000 mon.a (mon.0) 1408 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-07-28T19:47:42.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:41 smithi138 bash[67894]: audit 2024-07-28T19:47:40.558541+0000 mon.a (mon.0) 1409 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-07-28T19:47:42.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:41 smithi138 bash[67894]: audit 2024-07-28T19:47:40.558706+0000 mon.a (mon.0) 1410 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 7]}]': finished 2024-07-28T19:47:42.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:41 smithi138 bash[67894]: audit 2024-07-28T19:47:40.558840+0000 mon.a (mon.0) 1411 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]': finished 2024-07-28T19:47:42.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:41 smithi138 bash[67894]: cluster 2024-07-28T19:47:40.559054+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-07-28T19:47:42.581 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:47:42 smithi138 bash[60615]: ts=2024-07-28T19:47:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:47:42.861 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:42 smithi100 bash[80199]: cluster 2024-07-28T19:47:41.561243+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-07-28T19:47:42.861 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:42 smithi100 bash[80199]: cluster 2024-07-28T19:47:42.038109+0000 mgr.y (mgr.25358) 453 : cluster [DBG] pgmap v931: 105 pgs: 105 active+clean; 305 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 27 MiB/s wr, 194 op/s 2024-07-28T19:47:42.862 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:42 smithi100 bash[82644]: cluster 2024-07-28T19:47:41.561243+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-07-28T19:47:42.862 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:42 smithi100 bash[82644]: cluster 2024-07-28T19:47:42.038109+0000 mgr.y (mgr.25358) 453 : cluster [DBG] pgmap v931: 105 pgs: 105 active+clean; 305 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 27 MiB/s wr, 194 op/s 2024-07-28T19:47:42.862 INFO:tasks.workunit.client.0.smithi100.stdout: i=8 exists [112226~13193967,16897474~3285069,20365656~3799516,26761270~13144782,41510874~7067813,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,141880579~2515951,147636173~9583580,161173157~3951100,166822548~5169376,174554202~4352940,188958486~8827548,201778320~5844812,209422996~4801946,216071836~12282465,229242050~1098587,232482673~3872453,236800222~9802864,246992343~7340716,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~10712203,320568422~11577005,337654823~1632958,341344605~6887593,351158777~7342949,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] wrote [7481273~2119565,20365656~3799516,27981601~7708946,41510874~7067813,50290591~2207245,109479491~5347846,166822548~5169376,246992343~7340716,338413875~873906,388167688~7109454] 2024-07-28T19:47:43.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:42 smithi138 bash[67894]: cluster 2024-07-28T19:47:41.561243+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-07-28T19:47:43.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:42 smithi138 bash[67894]: cluster 2024-07-28T19:47:42.038109+0000 mgr.y (mgr.25358) 453 : cluster [DBG] pgmap v931: 105 pgs: 105 active+clean; 305 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 27 MiB/s wr, 194 op/s 2024-07-28T19:47:43.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:47:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:47:42] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.43.0" 2024-07-28T19:47:43.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:43 smithi100 bash[80199]: cluster 2024-07-28T19:47:42.588907+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-07-28T19:47:43.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:43 smithi100 bash[82644]: cluster 2024-07-28T19:47:42.588907+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-07-28T19:47:44.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:43 smithi138 bash[67894]: cluster 2024-07-28T19:47:42.588907+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-07-28T19:47:44.714 INFO:tasks.workunit.client.0.smithi100.stdout: i=9 exists [112226~13193967,16897474~3285069,20365656~3799516,26761270~13144782,41510874~7067813,49157145~3421699,54264737~383997,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,161173157~4582160,166822548~5169376,174554202~4352940,179002214~2817982,188958486~8827548,201778320~5844812,209422996~4801946,216071836~12282465,229242050~1098587,232482673~3872453,236800222~9802864,246992343~7340716,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~10712203,320568422~11577005,337540878~10691320,351158777~14016336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] wrote [54264737~383997,71835562~1532304,139049388~2220464,161225466~4529851,179002214~2817982,211982304~909576,217934213~1252500,337540878~4423331,357603777~7571336,385030579~8097932] 2024-07-28T19:47:44.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:44 smithi100 bash[80199]: cluster 2024-07-28T19:47:43.596917+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-07-28T19:47:44.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:44 smithi100 bash[80199]: cluster 2024-07-28T19:47:44.039087+0000 mgr.y (mgr.25358) 454 : cluster [DBG] pgmap v934: 105 pgs: 1 peering, 104 active+clean; 380 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 23 MiB/s wr, 148 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-07-28T19:47:44.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:44 smithi100 bash[82644]: cluster 2024-07-28T19:47:43.596917+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-07-28T19:47:44.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:44 smithi100 bash[82644]: cluster 2024-07-28T19:47:44.039087+0000 mgr.y (mgr.25358) 454 : cluster [DBG] pgmap v934: 105 pgs: 1 peering, 104 active+clean; 380 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 23 MiB/s wr, 148 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-07-28T19:47:45.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:44 smithi138 bash[67894]: cluster 2024-07-28T19:47:43.596917+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-07-28T19:47:45.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:44 smithi138 bash[67894]: cluster 2024-07-28T19:47:44.039087+0000 mgr.y (mgr.25358) 454 : cluster [DBG] pgmap v934: 105 pgs: 1 peering, 104 active+clean; 380 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 23 MiB/s wr, 148 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-07-28T19:47:45.720 INFO:tasks.workunit.client.0.smithi100.stdout: i=10 exists [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~5761548,166822548~5169376,174554202~10978765,188958486~8827548,201778320~5844812,209422996~4801946,215999575~12354726,229242050~1098587,231922261~4432865,236800222~9802864,246992343~10133613,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~10712203,320568422~11577005,337540878~10691320,351158777~14016336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] wrote [26493107~2633969,45794658~7953332,86185784~1193178,92287280~7627963,159993769~1588339,177959428~7573539,215999575~1776727,231922261~4372786,253887767~3238189,391211106~2731407] 2024-07-28T19:47:45.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:45 smithi100 bash[80199]: cluster 2024-07-28T19:47:44.611777+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-07-28T19:47:45.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:45 smithi100 bash[82644]: cluster 2024-07-28T19:47:44.611777+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-07-28T19:47:46.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:45 smithi138 bash[67894]: cluster 2024-07-28T19:47:44.611777+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-07-28T19:47:46.701 INFO:tasks.workunit.client.0.smithi100.stdout: i=11 exists [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77096290~1863886,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~5761548,166822548~5169376,174554202~10978765,188958486~8827548,201778320~5844812,209422996~5818018,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~10712203,320568422~11577005,337540878~10691320,351158777~14016336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] wrote [20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,87027871~1807545,120515010~4331715,213375755~1865259,235936831~3095605,256606700~476669,355162073~1084006] 2024-07-28T19:47:46.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:46 smithi100 bash[80199]: cluster 2024-07-28T19:47:45.621284+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-07-28T19:47:46.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:46 smithi100 bash[80199]: cluster 2024-07-28T19:47:46.039818+0000 mgr.y (mgr.25358) 455 : cluster [DBG] pgmap v937: 105 pgs: 1 peering, 104 active+clean; 422 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 30 MiB/s wr, 151 op/s; 5.0 MiB/s, 2 keys/s, 4 objects/s recovering 2024-07-28T19:47:46.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:46 smithi100 bash[82644]: cluster 2024-07-28T19:47:45.621284+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-07-28T19:47:46.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:46 smithi100 bash[82644]: cluster 2024-07-28T19:47:46.039818+0000 mgr.y (mgr.25358) 455 : cluster [DBG] pgmap v937: 105 pgs: 1 peering, 104 active+clean; 422 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 30 MiB/s wr, 151 op/s; 5.0 MiB/s, 2 keys/s, 4 objects/s recovering 2024-07-28T19:47:47.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:46 smithi138 bash[67894]: cluster 2024-07-28T19:47:45.621284+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-07-28T19:47:47.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:46 smithi138 bash[67894]: cluster 2024-07-28T19:47:46.039818+0000 mgr.y (mgr.25358) 455 : cluster [DBG] pgmap v937: 105 pgs: 1 peering, 104 active+clean; 422 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 92 KiB/s rd, 30 MiB/s wr, 151 op/s; 5.0 MiB/s, 2 keys/s, 4 objects/s recovering 2024-07-28T19:47:47.736 INFO:tasks.workunit.client.0.smithi100.stdout: i=12 exists [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~11998155,174554202~10978765,188958486~8827548,201778320~5844812,207679486~7561528,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~20466366,320568422~11577005,337540878~12988822,351158777~14016336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] wrote [3636392~2148951,77036241~6394957,149537883~2616756,163865563~5420861,176051091~3643688,189908632~4104040,207679486~2890179,304100381~6208439,311478804~409659,346870513~3659187] 2024-07-28T19:47:47.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:47 smithi100 bash[80199]: cluster 2024-07-28T19:47:46.637747+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-07-28T19:47:47.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:47 smithi100 bash[82644]: cluster 2024-07-28T19:47:46.637747+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-07-28T19:47:48.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:47 smithi138 bash[67894]: cluster 2024-07-28T19:47:46.637747+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-07-28T19:47:48.757 INFO:tasks.workunit.client.0.smithi100.stdout: i=13 exists [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,141880579~2515951,147636173~9583580,159993769~12518383,174554202~10978765,188958486~8827548,201778320~5844812,207679486~7561528,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~20466366,320568422~12470896,337540878~12988822,351158777~20513665,381707621~13569521,395829986~13298434,409516070~3951114] wrote [108462095~3696521,113389514~7808033,127746465~6902182,162125061~10387091,176420840~1173598,233701664~1501648,330570804~2468514,364820588~1893967] 2024-07-28T19:47:48.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:48 smithi100 bash[80199]: cluster 2024-07-28T19:47:47.637979+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-07-28T19:47:48.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:48 smithi100 bash[80199]: cluster 2024-07-28T19:47:48.040528+0000 mgr.y (mgr.25358) 456 : cluster [DBG] pgmap v940: 105 pgs: 1 peering, 104 active+clean; 440 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 15 MiB/s wr, 96 op/s; 2.6 MiB/s, 2 keys/s, 2 objects/s recovering 2024-07-28T19:47:48.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:48 smithi100 bash[82644]: cluster 2024-07-28T19:47:47.637979+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-07-28T19:47:48.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:48 smithi100 bash[82644]: cluster 2024-07-28T19:47:48.040528+0000 mgr.y (mgr.25358) 456 : cluster [DBG] pgmap v940: 105 pgs: 1 peering, 104 active+clean; 440 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 15 MiB/s wr, 96 op/s; 2.6 MiB/s, 2 keys/s, 2 objects/s recovering 2024-07-28T19:47:49.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:48 smithi138 bash[67894]: cluster 2024-07-28T19:47:47.637979+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-07-28T19:47:49.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:48 smithi138 bash[67894]: cluster 2024-07-28T19:47:48.040528+0000 mgr.y (mgr.25358) 456 : cluster [DBG] pgmap v940: 105 pgs: 1 peering, 104 active+clean; 440 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 15 MiB/s wr, 96 op/s; 2.6 MiB/s, 2 keys/s, 2 objects/s recovering 2024-07-28T19:47:49.780 INFO:tasks.workunit.client.0.smithi100.stdout: i=14 exists [112226~13193967,16897474~3285069,20365656~3799516,26493107~27254883,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159993769~25539198,188958486~8827548,201071067~625261,201778320~5844812,207679486~7561528,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~20466366,320568422~12470896,337540878~12988822,351158777~20513665,381707621~13569521,395829986~13298434,409516070~3951114] wrote [38624001~3004469,98333376~1495812,108254853~6235284,139059271~4848819,171959564~2961421,201071067~625261,363707150~7670438,399392058~7026463] 2024-07-28T19:47:49.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:49 smithi100 bash[80199]: cluster 2024-07-28T19:47:48.662567+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-07-28T19:47:49.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:49 smithi100 bash[82644]: cluster 2024-07-28T19:47:48.662567+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-07-28T19:47:50.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:49 smithi138 bash[67894]: cluster 2024-07-28T19:47:48.662567+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-07-28T19:47:50.777 INFO:tasks.workunit.client.0.smithi100.stdout: i=15 exists [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159680442~25852525,188958486~8827548,201071067~625261,201778320~5844812,207679486~7561528,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,274828887~893301,276135342~10754911,287356099~7126278,299052751~20466366,320568422~12470896,337540878~12988822,351158777~20513665,381707621~27420799,409516070~3951114] wrote [992284~7616291,11026132~4097000,29732490~296310,53284707~5794772,84756138~4620842,159680442~5214158,308197699~851853,360999874~2416421,393439548~6441384] 2024-07-28T19:47:50.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:50 smithi100 bash[80199]: cluster 2024-07-28T19:47:49.659985+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-07-28T19:47:50.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:50 smithi100 bash[80199]: cluster 2024-07-28T19:47:50.041252+0000 mgr.y (mgr.25358) 457 : cluster [DBG] pgmap v943: 105 pgs: 105 active+clean; 536 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 28 MiB/s wr, 288 op/s 2024-07-28T19:47:50.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:50 smithi100 bash[82644]: cluster 2024-07-28T19:47:49.659985+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-07-28T19:47:50.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:50 smithi100 bash[82644]: cluster 2024-07-28T19:47:50.041252+0000 mgr.y (mgr.25358) 457 : cluster [DBG] pgmap v943: 105 pgs: 105 active+clean; 536 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 28 MiB/s wr, 288 op/s 2024-07-28T19:47:51.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:50 smithi138 bash[67894]: cluster 2024-07-28T19:47:49.659985+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-07-28T19:47:51.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:50 smithi138 bash[67894]: cluster 2024-07-28T19:47:50.041252+0000 mgr.y (mgr.25358) 457 : cluster [DBG] pgmap v943: 105 pgs: 105 active+clean; 536 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 28 MiB/s wr, 288 op/s 2024-07-28T19:47:51.809 INFO:tasks.workunit.client.0.smithi100.stdout: i=16 exists [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159680442~25852525,188958486~8827548,201071067~625261,201778320~5844812,207679486~7561528,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,272948381~1516685,274828887~893301,276135342~10754911,287356099~7126278,299052751~20466366,320568422~13509234,337540878~13118270,351158777~20513665,381707621~27420799,409516070~3951114] wrote [67707114~4813819,101072762~6550905,176163511~5783392,272948381~1516685,288553325~3571025,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:51.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:51 smithi100 bash[80199]: cluster 2024-07-28T19:47:50.683110+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-07-28T19:47:51.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:51 smithi100 bash[82644]: cluster 2024-07-28T19:47:50.683110+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-07-28T19:47:52.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:51 smithi138 bash[67894]: cluster 2024-07-28T19:47:50.683110+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-07-28T19:47:52.702 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:47:52 smithi138 bash[60615]: ts=2024-07-28T19:47:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:47:52.809 INFO:tasks.workunit.client.0.smithi100.stdout: i=17 exists [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159680442~25852525,188958486~8827548,201071067~625261,201778320~13462694,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,272948381~1516685,274828887~893301,275837321~11052932,287356099~7126278,299052751~20466366,320568422~13509234,337540878~13118270,351158777~20513665,381707621~27420799,409516070~3951114] wrote [4007687~7504201,11782144~1642460,53164013~536842,56120723~5359237,94703460~1486531,142610709~6042503,205246723~4536920,239968511~70796,275837321~6859657] 2024-07-28T19:47:52.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:47:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:47:52] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2024-07-28T19:47:52.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:52 smithi100 bash[80199]: cluster 2024-07-28T19:47:51.697370+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-07-28T19:47:52.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:52 smithi100 bash[80199]: cluster 2024-07-28T19:47:52.041949+0000 mgr.y (mgr.25358) 458 : cluster [DBG] pgmap v946: 105 pgs: 105 active+clean; 553 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 115 KiB/s rd, 27 MiB/s wr, 199 op/s 2024-07-28T19:47:52.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:52 smithi100 bash[80199]: audit 2024-07-28T19:47:52.642870+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:52.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:52 smithi100 bash[82644]: cluster 2024-07-28T19:47:51.697370+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-07-28T19:47:52.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:52 smithi100 bash[82644]: cluster 2024-07-28T19:47:52.041949+0000 mgr.y (mgr.25358) 458 : cluster [DBG] pgmap v946: 105 pgs: 105 active+clean; 553 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 115 KiB/s rd, 27 MiB/s wr, 199 op/s 2024-07-28T19:47:52.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:52 smithi100 bash[82644]: audit 2024-07-28T19:47:52.642870+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:53.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:52 smithi138 bash[67894]: cluster 2024-07-28T19:47:51.697370+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-07-28T19:47:53.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:52 smithi138 bash[67894]: cluster 2024-07-28T19:47:52.041949+0000 mgr.y (mgr.25358) 458 : cluster [DBG] pgmap v946: 105 pgs: 105 active+clean; 553 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 115 KiB/s rd, 27 MiB/s wr, 199 op/s 2024-07-28T19:47:53.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:52 smithi138 bash[67894]: audit 2024-07-28T19:47:52.642870+0000 mon.a (mon.0) 1424 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:47:53.824 INFO:tasks.workunit.client.0.smithi100.stdout: i=18 exists [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159680442~25852525,188958486~8827548,201071067~625261,201778320~13462694,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,272948381~1516685,274828887~893301,275837321~11052932,287356099~32163018,320568422~13509234,337540878~13118270,351158777~20513665,381707621~27420799,409516070~3951114] wrote [42863566~319415,50422011~6169396,60101578~2634457,141112044~3631280,174132279~2675167,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566] 2024-07-28T19:47:53.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:53 smithi100 bash[80199]: cluster 2024-07-28T19:47:52.708896+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-07-28T19:47:53.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:53 smithi100 bash[82644]: cluster 2024-07-28T19:47:52.708896+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-07-28T19:47:54.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:53 smithi138 bash[67894]: cluster 2024-07-28T19:47:52.708896+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-07-28T19:47:54.825 INFO:tasks.workunit.client.0.smithi100.stdout: i=19 exists [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~8827548,201071067~625261,201778320~13462694,215999575~12354726,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,270155517~2642034,272948381~1516685,274828887~893301,275837321~11052932,287356099~32163018,320568422~13509234,337540878~13118270,351158777~20513665,375306772~1084165,381707621~27420799,409516070~3951114] wrote [13965872~2199139,35955503~22983,61441917~13279843,74931225~3352106,86448622~4240801,108627228~3624510,159391129~1478312,178193457~4505245,375306772~1084165] 2024-07-28T19:47:54.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:54 smithi100 bash[80199]: cluster 2024-07-28T19:47:53.715166+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-07-28T19:47:54.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:54 smithi100 bash[80199]: cluster 2024-07-28T19:47:54.042744+0000 mgr.y (mgr.25358) 459 : cluster [DBG] pgmap v949: 105 pgs: 105 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 229 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-07-28T19:47:54.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:54 smithi100 bash[82644]: cluster 2024-07-28T19:47:53.715166+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-07-28T19:47:54.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:54 smithi100 bash[82644]: cluster 2024-07-28T19:47:54.042744+0000 mgr.y (mgr.25358) 459 : cluster [DBG] pgmap v949: 105 pgs: 105 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 229 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-07-28T19:47:55.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:54 smithi138 bash[67894]: cluster 2024-07-28T19:47:53.715166+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-07-28T19:47:55.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:54 smithi138 bash[67894]: cluster 2024-07-28T19:47:54.042744+0000 mgr.y (mgr.25358) 459 : cluster [DBG] pgmap v949: 105 pgs: 105 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 229 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-07-28T19:47:55.750 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 1 diff [11015273~2290920,32983006~6923046,58061600~4881219,66565039~8114072,141880579~2515951,147636173~6380355,241226272~5376814,328798819~3346608,366220121~5452321] 2024-07-28T19:47:55.750 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [11015273~2290920,32983006~6923046,58061600~4881219,66565039~8114072,141880579~2515951,147636173~6380355,241226272~5376814,328798819~3346608,366220121~5452321] 2024-07-28T19:47:55.788 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4917585,29360128~10545924,54525952~20153159,138412032~5984498,146800640~7215888,239075328~7527758,327155712~4989715,364904448~6767994] 2024-07-28T19:47:55.789 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 2 diff [11015273~2290920,26761270~3620690,32983006~6923046,58061600~4881219,66565039~8114072,141880579~2515951,147636173~6380355,161173157~3951100,229242050~1098587,236800222~9802864,303607713~4909202,328798819~3346608,351158777~5921015,366220121~5452321,409516070~3951114] 2024-07-28T19:47:55.789 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [11015273~2290920,26761270~3620690,32983006~6923046,58061600~4881219,66565039~8114072,141880579~2515951,147636173~6380355,161173157~3951100,229242050~1098587,236800222~9802864,303607713~4909202,328798819~3346608,351158777~5921015,366220121~5452321,409516070~3951114] 2024-07-28T19:47:55.804 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4917585,25165824~14740228,54525952~20153159,138412032~5984498,146800640~7215888,159383552~5740705,228354301~1986336,234881024~11722062,303607713~4909202,327155712~4989715,348127232~8952560,364904448~6767994,407282580~6184604] 2024-07-28T19:47:55.804 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 3 diff [11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,58061600~4881219,66565039~8114072,77096290~1863886,141880579~2515951,147636173~6380355,161173157~3951100,167752029~510497,229242050~1098587,236800222~9802864,281629645~5260608,288011974~6470403,299052751~9464164,326954272~5191155,343359455~4872743,351158777~5921015,366220121~5452321,409516070~3951114] 2024-07-28T19:47:55.804 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,58061600~4881219,66565039~8114072,77096290~1863886,141880579~2515951,147636173~6380355,161173157~3951100,167752029~510497,229242050~1098587,236800222~9802864,281629645~5260608,288011974~6470403,299052751~9464164,326954272~5191155,343359455~4872743,351158777~5921015,366220121~5452321,409516070~3951114] 2024-07-28T19:47:55.819 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4917585,25165824~14740228,46137344~6441500,54525952~20153159,75497472~3462704,138412032~5984498,146800640~7215888,159383552~8878974,228354301~1986336,234881024~11722062,281018368~13464009,297795584~10721331,322961408~9184019,343359455~13720337,364904448~6767994,407282580~6184604] 2024-07-28T19:47:55.819 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 4 diff [11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,58061600~4881219,66565039~8114072,77096290~1863886,141880579~2515951,147636173~6380355,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,216071836~7254506,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,326954272~5191155,337654823~1447438,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,409516070~3951114] 2024-07-28T19:47:55.819 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,58061600~4881219,66565039~8114072,77096290~1863886,141880579~2515951,147636173~6380355,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,216071836~7254506,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,326954272~5191155,337654823~1447438,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,409516070~3951114] 2024-07-28T19:47:55.832 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4917585,25165824~14740228,46137344~6441500,54525952~20153159,75497472~3462704,138412032~5984498,146800640~7215888,159383552~8878974,188958486~1816264,201326592~6296540,213909504~9416838,228354301~1986336,230686720~15916366,247463936~6003165,271270161~8790119,281018368~13464009,297795584~10721331,322961408~9184019,335544320~3557941,343359455~13720337,364904448~6767994,381681664~5406250,407282580~6184604] 2024-07-28T19:47:55.832 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 5 diff [112226~7216589,11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~7254506,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,326954272~5191155,337654823~1447438,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.832 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~7216589,11015273~2290920,26761270~3620690,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~7254506,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,326954272~5191155,337654823~1447438,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.843 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~7328815,8388608~4917585,25165824~14740228,46137344~6441500,54525952~20153159,75497472~3462704,88080384~2503854,138412032~5984498,146800640~10419113,159383552~8878974,188958486~1816264,201326592~21999750,228354301~1986336,230686720~15916366,247463936~6003165,271270161~8790119,281018368~13464009,297795584~10721331,322961408~9184019,335544320~3557941,343359455~13720337,364904448~6767994,381681664~5406250,394264576~19202608] 2024-07-28T19:47:55.843 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 6 diff [112226~13193967,16897474~3285069,26761270~3620690,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,264356515~4168024,271270161~1527390,276135342~3924938,281629645~5260608,287356099~7126278,299052751~9464164,320568422~11577005,337654823~1447438,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.843 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,26761270~3620690,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,264356515~4168024,271270161~1527390,276135342~3924938,281629645~5260608,287356099~7126278,299052751~9464164,320568422~11577005,337654823~1447438,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.853 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~3405327,25165824~14740228,46137344~6441500,54525952~20153159,75497472~3462704,88080384~2503854,117440512~2794423,138412032~5984498,146800640~10419113,159383552~8878974,188958486~1816264,201326592~23856330,228354301~1986336,230686720~15916366,247463936~6003165,264241152~4283387,271270161~8790119,281018368~13464009,297795584~10721331,319519117~12626310,335544320~3557941,343359455~13720337,364904448~6767994,381681664~5406250,394264576~19202608] 2024-07-28T19:47:55.853 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 7 diff [112226~13193967,16897474~3285069,26761270~3896241,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,126138082~3957397,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337654823~1447438,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.853 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,26761270~3896241,31967198~7938854,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,126138082~3957397,141880579~2515951,147636173~9583580,161173157~3951100,167752029~510497,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~9802864,250280585~3186516,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337654823~1447438,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.863 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~3405327,25165824~14740228,46137344~6441500,54525952~20153159,75497472~3462704,88080384~2503854,117440512~2794423,125829120~4266359,138412032~5984498,146800640~10419113,159383552~8878974,171966464~6940678,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~15916366,247463936~6003165,260046848~8477691,271270161~23212216,297795584~18618875,319519117~12626310,335544320~3557941,343359455~13720337,364904448~6767994,381681664~5406250,394264576~19202608] 2024-07-28T19:47:55.863 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 8 diff [112226~13193967,16897474~3285069,20365656~3799516,26761270~13144782,41510874~7067813,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,141880579~2515951,147636173~9583580,161173157~3951100,166822548~5169376,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~9802864,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337654823~1632958,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,388167688~7109454,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.863 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,20365656~3799516,26761270~13144782,41510874~7067813,49157145~3421699,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,141880579~2515951,147636173~9583580,161173157~3951100,166822548~5169376,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~9802864,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337654823~1632958,343359455~4872743,351158777~5921015,366220121~5452321,381707621~5380293,388167688~7109454,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.872 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~7387956,25165824~27413020,54525952~20153159,75497472~3462704,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,138412032~5984498,146800640~10419113,159383552~19523590,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~23646339,260046848~8477691,271270161~23212216,297795584~18618875,319519117~12626310,335544320~3743461,343359455~13720337,364904448~6767994,381681664~5406250,388167688~25299496] 2024-07-28T19:47:55.872 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 9 diff [112226~13193967,16897474~3285069,20365656~3799516,26761270~13144782,41510874~7067813,49157145~3421699,54264737~383997,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,161173157~4582160,166822548~5169376,174554202~4352940,179002214~2817982,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~9802864,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.872 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,20365656~3799516,26761270~13144782,41510874~7067813,49157145~3421699,54264737~383997,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,161173157~4582160,166822548~5169376,174554202~4352940,179002214~2817982,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~9802864,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.881 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~7387956,25165824~49513287,75497472~3462704,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,138412032~5984498,146800640~10419113,159383552~22436644,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~23646339,260046848~8477691,271270161~23212216,297795584~18618875,319519117~12626310,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.881 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 10 diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~4801946,215999575~9183347,225682819~1876237,229242050~1098587,231922261~4432865,236800222~9802864,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.881 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55908035~980737,58061600~4881219,65261141~9417970,75987387~726387,77096290~1863886,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~4801946,215999575~9183347,225682819~1876237,229242050~1098587,231922261~4432865,236800222~9802864,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.890 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~7387956,25165824~49513287,75497472~3462704,83886080~3492882,88080384~2503854,92274688~7640555,109051904~5775433,117440512~2794423,125829120~4266359,138412032~5984498,146800640~10419113,159383552~26149415,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~26439236,260046848~8477691,271270161~23212216,297795584~18618875,319519117~12626310,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.890 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 11 diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77096290~1863886,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~5818018,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.891 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77096290~1863886,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~5818018,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~11577005,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.898 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~7387956,25165824~49513287,75497472~3462704,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,138412032~5984498,146800640~10419113,159383552~26149415,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~26439236,260046848~8477691,271270161~23212216,297795584~18618875,319519117~12626310,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.898 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 12 diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~11998155,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~11577005,337540878~4423331,343359455~7170245,351158777~5921015,357603777~7571336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.898 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,141880579~2515951,147636173~9583580,159993769~11998155,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~11577005,337540878~4423331,343359455~7170245,351158777~5921015,357603777~7571336,366220121~5452321,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.906 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~7387956,25165824~49513287,75497472~7933726,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,138412032~5984498,146800640~10419113,159383552~26149415,188958486~5054186,201326592~23856330,225682819~1876237,228354301~1986336,230686720~26439236,260046848~8477691,271270161~23212216,297795584~18618875,319519117~12626310,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.906 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 13 diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,141880579~2515951,147636173~9583580,159993769~12518383,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~12470896,337540878~4423331,343359455~7170245,351158777~5921015,357603777~14068665,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.906 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~13412945,41510874~12237116,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,141880579~2515951,147636173~9583580,159993769~12518383,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~12470896,337540878~4423331,343359455~7170245,351158777~5921015,357603777~14068665,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.913 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~7387956,25165824~49513287,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,138412032~5984498,146800640~10419113,159383552~26149415,188958486~5054186,201326592~23856330,225682819~1876237,228354301~1986336,230686720~26439236,260046848~8477691,271270161~23212216,297795584~18618875,319519117~13520201,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.913 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 14 diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~27254883,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159993769~25539198,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~12470896,337540878~4423331,343359455~7170245,351158777~5921015,357603777~14068665,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.913 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13193967,16897474~3285069,20365656~3799516,26493107~27254883,54264737~383997,55289528~7653291,65261141~9417970,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159993769~25539198,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~12470896,337540878~4423331,343359455~7170245,351158777~5921015,357603777~14068665,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.921 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13306193,16777216~7387956,25165824~49513287,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,138412032~5984498,146800640~10419113,159383552~26149415,188958486~5054186,197786034~27396888,225682819~1876237,228354301~1986336,230686720~26439236,260046848~8477691,271270161~23212216,297795584~18618875,319519117~13520201,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.921 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 15 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~12470896,337540878~4423331,343359455~7170245,351158777~5921015,357603777~14068665,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.921 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~12470896,337540878~4423331,343359455~7170245,351158777~5921015,357603777~14068665,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.928 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,25165824~49513287,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,138412032~5984498,146800640~10419113,159383552~26149415,188958486~5054186,197786034~27396888,225682819~1876237,228354301~1986336,230686720~26439236,260046848~8477691,271270161~23212216,297795584~18618875,319519117~13520201,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.928 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 16 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.928 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~5347142,147636173~9583580,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.936 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,25165824~49513287,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,138412032~5984498,146800640~10419113,159383552~26149415,188958486~5054186,197786034~27396888,225682819~1876237,228354301~1986336,230686720~26439236,260046848~8477691,271270161~23212216,297795584~18618875,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.936 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 17 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.936 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~9183347,225682819~1876237,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.942 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,25165824~49513287,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~27396888,225682819~1876237,228354301~1986336,230686720~26439236,260046848~8477691,271270161~23212216,297795584~18618875,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.943 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 18 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.943 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~36449712,65261141~9417970,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.949 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,25165824~49513287,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,381681664~31785520] 2024-07-28T19:47:55.949 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 19 diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.949 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:55.956 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:47:55.956 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 2 diff [26761270~3620690,68867624~4203403,161173157~3951100,229242050~1098587,236800222~4533966,303607713~4909202,351158777~5921015,409516070~3951114] 2024-07-28T19:47:55.957 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [26761270~3620690,68867624~4203403,161173157~3951100,229242050~1098587,236800222~4533966,303607713~4909202,351158777~5921015,409516070~3951114] 2024-07-28T19:47:55.963 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [25165824~5216136,68867624~4203403,159383552~5740705,228354301~1986336,234881024~6453164,303607713~4909202,348127232~8952560,407282580~6184604] 2024-07-28T19:47:55.963 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 3 diff [26761270~3620690,31967198~1848860,49157145~3421699,68867624~4203403,77096290~1863886,161173157~3951100,167752029~510497,229242050~1098587,236800222~4533966,281629645~5260608,288011974~6470403,299052751~9464164,326954272~2883588,343359455~4872743,351158777~5921015,409516070~3951114] 2024-07-28T19:47:55.964 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [26761270~3620690,31967198~1848860,49157145~3421699,68867624~4203403,77096290~1863886,161173157~3951100,167752029~510497,229242050~1098587,236800222~4533966,281629645~5260608,288011974~6470403,299052751~9464164,326954272~2883588,343359455~4872743,351158777~5921015,409516070~3951114] 2024-07-28T19:47:55.971 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [25165824~5216136,31967198~1848860,46137344~6441500,68867624~4203403,75497472~3462704,159383552~8878974,228354301~1986336,234881024~6453164,281018368~13464009,297795584~10721331,322961408~6876452,343359455~13720337,407282580~6184604] 2024-07-28T19:47:55.971 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 4 diff [26761270~3620690,31967198~1848860,49157145~3421699,68867624~4203403,77096290~1863886,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,216071836~7254506,229242050~1098587,232482673~3872453,236800222~4533966,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,326954272~2883588,337654823~1447438,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,409516070~3951114] 2024-07-28T19:47:55.971 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [26761270~3620690,31967198~1848860,49157145~3421699,68867624~4203403,77096290~1863886,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,216071836~7254506,229242050~1098587,232482673~3872453,236800222~4533966,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,326954272~2883588,337654823~1447438,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,409516070~3951114] 2024-07-28T19:47:55.978 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [25165824~5216136,31967198~1848860,46137344~6441500,68867624~4203403,75497472~3462704,159383552~8878974,188958486~1816264,201326592~6296540,213909504~9416838,228354301~1986336,230686720~10647468,247463936~6003165,271270161~8790119,281018368~13464009,297795584~10721331,322961408~6876452,335544320~3557941,343359455~13720337,367354362~1092722,381681664~5406250,407282580~6184604] 2024-07-28T19:47:55.978 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 5 diff [112226~7216589,26761270~3620690,31967198~1848860,49157145~3421699,55908035~980737,65261141~7809886,75987387~726387,77096290~1863886,90327583~256655,152843223~4376530,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~7254506,229242050~1098587,232482673~3872453,236800222~4533966,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,326954272~2883588,337654823~1447438,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.978 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~7216589,26761270~3620690,31967198~1848860,49157145~3421699,55908035~980737,65261141~7809886,75987387~726387,77096290~1863886,90327583~256655,152843223~4376530,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~7254506,229242050~1098587,232482673~3872453,236800222~4533966,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~9464164,326954272~2883588,337654823~1447438,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.985 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~7328815,25165824~5216136,31967198~1848860,46137344~6441500,55908035~980737,65261141~7809886,75497472~3462704,88080384~2503854,152843223~4376530,159383552~8878974,188958486~1816264,201326592~21999750,228354301~1986336,230686720~10647468,247463936~6003165,271270161~8790119,281018368~13464009,297795584~10721331,322961408~6876452,335544320~3557941,343359455~13720337,367354362~1092722,381681664~5406250,394264576~19202608] 2024-07-28T19:47:55.985 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 6 diff [112226~13187071,16897474~3285069,26761270~3620690,31967198~1848860,49157145~3421699,55908035~980737,65261141~7809886,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,152843223~4376530,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,229242050~1098587,232482673~3872453,236800222~4533966,250280585~3186516,264356515~4168024,271270161~1527390,276135342~3924938,281629645~5260608,287356099~7126278,299052751~9464164,320568422~9269438,337654823~1447438,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.985 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,26761270~3620690,31967198~1848860,49157145~3421699,55908035~980737,65261141~7809886,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,152843223~4376530,161173157~3951100,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,229242050~1098587,232482673~3872453,236800222~4533966,250280585~3186516,264356515~4168024,271270161~1527390,276135342~3924938,281629645~5260608,287356099~7126278,299052751~9464164,320568422~9269438,337654823~1447438,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.991 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~3405327,25165824~5216136,31967198~1848860,46137344~6441500,55908035~980737,65261141~7809886,75497472~3462704,88080384~2503854,117440512~2794423,152843223~4376530,159383552~8878974,188958486~1816264,201326592~23856330,228354301~1986336,230686720~10647468,247463936~6003165,264241152~4283387,271270161~8790119,281018368~13464009,297795584~10721331,319519117~10318743,335544320~3557941,343359455~13720337,367354362~1092722,381681664~5406250,394264576~19202608] 2024-07-28T19:47:55.991 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 7 diff [112226~13187071,16897474~3285069,26761270~3896241,31967198~1848860,49157145~3421699,55908035~980737,65261141~7809886,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,126138082~3957397,152843223~4376530,161173157~3951100,167752029~510497,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~4533966,250280585~3186516,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337654823~1447438,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.991 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,26761270~3896241,31967198~1848860,49157145~3421699,55908035~980737,65261141~7809886,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,126138082~3957397,152843223~4376530,161173157~3951100,167752029~510497,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~4533966,250280585~3186516,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337654823~1447438,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:55 smithi100 bash[80199]: cluster 2024-07-28T19:47:54.731512+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-07-28T19:47:55.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:55 smithi100 bash[82644]: cluster 2024-07-28T19:47:54.731512+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-07-28T19:47:55.999 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~3405327,25165824~5491687,31967198~1848860,46137344~6441500,55908035~980737,65261141~7809886,75497472~3462704,88080384~2503854,117440512~2794423,125829120~4266359,152843223~4376530,159383552~8878974,171966464~6940678,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~10647468,247463936~6003165,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,335544320~3557941,343359455~13720337,367354362~1092722,381681664~5406250,394264576~19202608] 2024-07-28T19:47:55.999 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 8 diff [112226~13187071,16897474~3285069,20365656~3799516,26761270~8929277,41510874~7067813,49157145~3421699,55908035~980737,65261141~7809886,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,152843223~4376530,161173157~3951100,166822548~5169376,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~4533966,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337654823~1632958,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,388167688~7109454,395829986~13298434,409516070~3951114] 2024-07-28T19:47:55.999 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26761270~8929277,41510874~7067813,49157145~3421699,55908035~980737,65261141~7809886,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,152843223~4376530,161173157~3951100,166822548~5169376,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~4533966,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337654823~1632958,343359455~4872743,351158777~5921015,367354362~1092722,381707621~5380293,388167688~7109454,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.005 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,25165824~10524723,39906052~12672792,55908035~980737,65261141~7809886,75497472~3462704,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,152843223~4376530,159383552~19523590,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~10647468,246603086~7729973,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,335544320~3743461,343359455~13720337,367354362~1092722,381681664~5406250,388167688~25299496] 2024-07-28T19:47:56.005 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 9 diff [112226~13187071,16897474~3285069,20365656~3799516,26761270~8929277,41510874~7067813,49157145~3421699,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,161173157~4582160,166822548~5169376,174554202~4352940,179002214~2817982,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~4533966,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.005 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26761270~8929277,41510874~7067813,49157145~3421699,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,161173157~4582160,166822548~5169376,174554202~4352940,179002214~2817982,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,229242050~1098587,232482673~3872453,236800222~4533966,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.012 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,25165824~10524723,39906052~14742682,55908035~980737,65261141~8106725,75497472~3462704,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,152843223~4376530,159383552~22436644,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~10647468,246603086~7729973,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,335544320~6419889,343359455~13720337,357603777~7571336,367354362~1092722,381681664~31785520] 2024-07-28T19:47:56.012 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 10 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,77096290~1863886,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~4801946,215999575~9183347,225682819~1876237,229242050~1098587,231922261~4432865,236800222~4533966,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.012 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,77096290~1863886,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~4801946,215999575~9183347,225682819~1876237,229242050~1098587,231922261~4432865,236800222~4533966,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.018 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,25165824~10524723,39906052~14742682,55908035~980737,65261141~8106725,75497472~3462704,83886080~3492882,88080384~2503854,92274688~7640555,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,152843223~4376530,159383552~26149415,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,335544320~6419889,343359455~13720337,357603777~7571336,367354362~1092722,381681664~31785520] 2024-07-28T19:47:56.019 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 11 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77096290~1863886,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~5818018,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.019 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77096290~1863886,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~5818018,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~9464164,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,351158777~5921015,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.025 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,25165824~10524723,39906052~14742682,55289528~3376295,65261141~8106725,75497472~3462704,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,152843223~4376530,159383552~26149415,188958486~1816264,201326592~23856330,225682819~1876237,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,335544320~6419889,343359455~13720337,357603777~7571336,367354362~1092722,381681664~31785520] 2024-07-28T19:47:56.025 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 12 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,152843223~4376530,159993769~11998155,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,337540878~4423331,343359455~7170245,351158777~5921015,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.025 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,152843223~4376530,159993769~11998155,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,337540878~4423331,343359455~7170245,351158777~5921015,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.033 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,25165824~10524723,39906052~14742682,55289528~3376295,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,201326592~23856330,225682819~1876237,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,335544320~6419889,343359455~13720337,357603777~7571336,367354362~1092722,381681664~31785520] 2024-07-28T19:47:56.033 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 13 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,152843223~4376530,159993769~12518383,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,351158777~5921015,357603777~9110778,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.033 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,152843223~4376530,159993769~12518383,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,351158777~5921015,357603777~9110778,367354362~1092722,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.041 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,25165824~10524723,39906052~14742682,55289528~3376295,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~2220464,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,201326592~23856330,225682819~1876237,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,330570804~2468514,335544320~6419889,343359455~13720337,357603777~9110778,367354362~1092722,381681664~31785520] 2024-07-28T19:47:56.041 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 14 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,351158777~5921015,357603777~13773811,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.041 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,351158777~5921015,357603777~13773811,381707621~13569521,395829986~13298434,409516070~3951114] 2024-07-28T19:47:56.047 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,25165824~10524723,38624001~16024733,55289528~3376295,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~27396888,225682819~1876237,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,330570804~2468514,335544320~6419889,343359455~13720337,357603777~13773811,381681664~31785520] 2024-07-28T19:47:56.047 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 15 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,351158777~5921015,357603777~13773811,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.047 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,351158777~5921015,357603777~13773811,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.054 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,25165824~10524723,38624001~20455478,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~27396888,225682819~1876237,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,330570804~2468514,335544320~6419889,343359455~13720337,357603777~13773811,381681664~31785520] 2024-07-28T19:47:56.054 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 16 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.055 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:55 smithi138 bash[67894]: cluster 2024-07-28T19:47:54.731512+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-07-28T19:47:56.062 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,25165824~10524723,38624001~20455478,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~27396888,225682819~1876237,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,381681664~31785520] 2024-07-28T19:47:56.062 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 17 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.062 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~9183347,225682819~1876237,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.069 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,25165824~10524723,38624001~22855959,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~27396888,225682819~1876237,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,381681664~31785520] 2024-07-28T19:47:56.070 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 18 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.070 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.076 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,25165824~10524723,38624001~24112034,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,381681664~31785520] 2024-07-28T19:47:56.076 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 19 diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.076 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:56.082 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:47:56.082 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 3 diff [31967198~1848860,49157145~3421699,77096290~1863886,167752029~510497,281629645~5260608,288011974~6470403,299052751~6022892,326954272~2883588,343359455~4872743] 2024-07-28T19:47:56.083 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [31967198~1848860,49157145~3421699,77096290~1863886,167752029~510497,281629645~5260608,288011974~6470403,299052751~6022892,326954272~2883588,343359455~4872743] 2024-07-28T19:47:56.089 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [31967198~1848860,46137344~6441500,75497472~3462704,165124257~3138269,281018368~13464009,297795584~7280059,322961408~6876452,343359455~4872743] 2024-07-28T19:47:56.089 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 4 diff [31967198~1848860,49157145~3421699,77096290~1863886,167752029~510497,188958486~1816264,201778320~5844812,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~6022892,326954272~2883588,337654823~1447438,343359455~4872743,367354362~1092722,381707621~5380293] 2024-07-28T19:47:56.089 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [31967198~1848860,49157145~3421699,77096290~1863886,167752029~510497,188958486~1816264,201778320~5844812,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~6022892,326954272~2883588,337654823~1447438,343359455~4872743,367354362~1092722,381707621~5380293] 2024-07-28T19:47:56.096 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [31967198~1848860,46137344~6441500,75497472~3462704,165124257~3138269,188958486~1816264,201326592~6296540,213909504~9416838,230686720~5668406,247463936~6003165,271270161~8790119,281018368~13464009,297795584~7280059,322961408~6876452,335544320~3557941,343359455~4872743,367354362~1092722,381681664~5406250] 2024-07-28T19:47:56.096 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 5 diff [112226~7216589,31967198~1848860,49157145~3421699,55908035~980737,65261141~6828665,75987387~726387,77096290~1863886,90327583~256655,152843223~4376530,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~6022892,326954272~2883588,337654823~1447438,343359455~4872743,367354362~1092722,381707621~5380293,395829986~13298434] 2024-07-28T19:47:56.096 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~7216589,31967198~1848860,49157145~3421699,55908035~980737,65261141~6828665,75987387~726387,77096290~1863886,90327583~256655,152843223~4376530,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,281629645~5260608,288011974~6470403,299052751~6022892,326954272~2883588,337654823~1447438,343359455~4872743,367354362~1092722,381707621~5380293,395829986~13298434] 2024-07-28T19:47:56.103 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~7328815,31967198~1848860,46137344~6441500,55908035~980737,65261141~6828665,75497472~3462704,88080384~2503854,152843223~4376530,165124257~3138269,188958486~1816264,201326592~21999750,230686720~5668406,247463936~6003165,271270161~8790119,281018368~13464009,297795584~7280059,322961408~6876452,335544320~3557941,343359455~4872743,367354362~1092722,381681664~5406250,394264576~14863844] 2024-07-28T19:47:56.103 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 6 diff [112226~13187071,16897474~3285069,31967198~1848860,49157145~3421699,55908035~980737,65261141~6828665,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,152843223~4376530,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,232482673~3872453,250280585~3186516,264356515~4168024,271270161~1527390,276135342~3924938,281629645~5260608,287356099~7126278,299052751~6022892,320568422~9269438,337654823~1447438,343359455~4872743,367354362~1092722,381707621~5380293,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.103 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,31967198~1848860,49157145~3421699,55908035~980737,65261141~6828665,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,152843223~4376530,167752029~510497,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,232482673~3872453,250280585~3186516,264356515~4168024,271270161~1527390,276135342~3924938,281629645~5260608,287356099~7126278,299052751~6022892,320568422~9269438,337654823~1447438,343359455~4872743,367354362~1092722,381707621~5380293,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.110 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~3405327,31967198~1848860,46137344~6441500,55908035~980737,65261141~6828665,75497472~3462704,88080384~2503854,117440512~2794423,152843223~4376530,165124257~3138269,188958486~1816264,201326592~23856330,230686720~5668406,247463936~6003165,264241152~4283387,271270161~8790119,281018368~13464009,297795584~7280059,319519117~10318743,335544320~3557941,343359455~4872743,367354362~1092722,381681664~5406250,394264576~14863844,410737048~1385636] 2024-07-28T19:47:56.110 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 7 diff [112226~13187071,16897474~3285069,27561189~3096322,31967198~1848860,49157145~3421699,55908035~980737,65261141~6828665,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,126138082~3957397,152843223~4376530,167752029~510497,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,250280585~3186516,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337654823~1447438,343359455~4872743,367354362~1092722,381707621~5380293,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.110 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,27561189~3096322,31967198~1848860,49157145~3421699,55908035~980737,65261141~6828665,75987387~726387,77096290~1863886,90327583~256655,117774156~2460779,126138082~3957397,152843223~4376530,167752029~510497,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,250280585~3186516,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337654823~1447438,343359455~4872743,367354362~1092722,381707621~5380293,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.116 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~3405327,27561189~3096322,31967198~1848860,46137344~6441500,55908035~980737,65261141~6828665,75497472~3462704,88080384~2503854,117440512~2794423,125829120~4266359,152843223~4376530,165124257~3138269,171966464~6940678,188958486~1816264,201326592~23856330,225682819~1876237,230686720~5668406,247463936~6003165,260046848~8477691,271270161~23212216,297795584~7280059,308516915~7897544,319519117~10318743,335544320~3557941,343359455~4872743,367354362~1092722,381681664~5406250,394264576~14863844,410737048~1385636] 2024-07-28T19:47:56.116 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 8 diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,49157145~3421699,55908035~980737,65261141~6828665,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,152843223~4376530,166822548~5169376,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337654823~1632958,343359455~4872743,367354362~1092722,381707621~5380293,388167688~7109454,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.116 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,49157145~3421699,55908035~980737,65261141~6828665,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,152843223~4376530,166822548~5169376,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337654823~1632958,343359455~4872743,367354362~1092722,381707621~5380293,388167688~7109454,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.123 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,27561189~8129358,39906052~12672792,55908035~980737,65261141~6828665,75497472~3462704,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,152843223~4376530,165124257~13782885,188958486~1816264,201326592~23856330,225682819~1876237,230686720~5668406,246603086~7729973,260046848~8477691,271270161~23212216,297795584~7280059,308516915~7897544,319519117~10318743,335544320~3743461,343359455~4872743,367354362~1092722,381681664~5406250,388167688~20960732,410737048~1385636] 2024-07-28T19:47:56.123 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 9 diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,49157145~3421699,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.123 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,49157145~3421699,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,77096290~1863886,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.130 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,27561189~8129358,39906052~14742682,55908035~980737,65261141~8106725,75497472~3462704,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,152843223~4376530,161225466~20594730,188958486~1816264,201326592~23856330,225682819~1876237,230686720~5668406,246603086~7729973,260046848~8477691,271270161~23212216,297795584~7280059,308516915~7897544,319519117~10318743,335544320~6419889,343359455~4872743,357603777~7571336,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.130 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 10 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,77096290~1863886,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~4801946,215999575~9183347,225682819~1876237,231922261~4432865,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.130 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,77096290~1863886,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~4801946,215999575~9183347,225682819~1876237,231922261~4432865,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.138 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55908035~980737,65261141~8106725,75497472~3462704,83886080~3492882,88080384~2503854,92274688~7640555,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,152843223~4376530,159993769~25539198,188958486~1816264,201326592~23856330,225682819~1876237,230686720~5668406,246603086~10522870,260046848~8477691,271270161~23212216,297795584~7280059,308516915~7897544,319519117~10318743,335544320~6419889,343359455~4872743,357603777~7571336,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.138 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 11 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77096290~1863886,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~5818018,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,355162073~1084006,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.138 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77096290~1863886,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~5818018,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~6022892,308806914~7607545,320568422~9269438,337540878~4423331,343359455~4872743,355162073~1084006,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.146 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75497472~3462704,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,152843223~4376530,159993769~25539198,188958486~1816264,201326592~23856330,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~23212216,297795584~7280059,308516915~7897544,319519117~10318743,335544320~6419889,343359455~4872743,355162073~1084006,357603777~7571336,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.146 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 12 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,152843223~4376530,159993769~11998155,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,337540878~4423331,343359455~7170245,355162073~1084006,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.146 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,152843223~4376530,159993769~11998155,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,337540878~4423331,343359455~7170245,355162073~1084006,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.152 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201326592~23856330,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,335544320~6419889,343359455~7170245,355162073~1084006,357603777~7571336,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.153 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 13 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,152843223~4376530,159993769~12518383,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,355162073~1084006,357603777~9110778,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.153 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,152843223~4376530,159993769~12518383,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,355162073~1084006,357603777~9110778,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.159 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~2220464,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201326592~23856330,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,330570804~2468514,335544320~6419889,343359455~7170245,355162073~1084006,357603777~9110778,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.159 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 14 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,355162073~1084006,357603777~13773811,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.159 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,355162073~1084006,357603777~13773811,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.165 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,38624001~16024733,55289528~3376295,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,197786034~27396888,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,330570804~2468514,335544320~6419889,343359455~7170245,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.165 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 15 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.166 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~9269438,330570804~2468514,337540878~4423331,343359455~7170245,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.172 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~20455478,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,197786034~27396888,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~10318743,330570804~2468514,335544320~6419889,343359455~7170245,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.172 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 16 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.173 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,276135342~10754911,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.179 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~20455478,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,197786034~27396888,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.180 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 17 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~9183347,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.180 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~9183347,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~7126278,299052751~17361708,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.187 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~22855959,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,197786034~27396888,225682819~1876237,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~23212216,297795584~18618875,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.187 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 18 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.187 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.194 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~24112034,65261141~8106725,75497472~7933726,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.194 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 19 diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.194 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.200 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.200 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 4 diff [188958486~1816264,201778320~5844812,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,337654823~1447438,367354362~1092722,381707621~5380293] 2024-07-28T19:47:56.200 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [188958486~1816264,201778320~5844812,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,337654823~1447438,367354362~1092722,381707621~5380293] 2024-07-28T19:47:56.207 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [188958486~1816264,201326592~6296540,213909504~9416838,230686720~5668406,247463936~6003165,271270161~8790119,335544320~3557941,367354362~1092722,381681664~5406250] 2024-07-28T19:47:56.208 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 5 diff [112226~7216589,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,152843223~4376530,188958486~1816264,201778320~5844812,209422996~4801946,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,337654823~1447438,367354362~1092722,381707621~5380293,395829986~13298434] 2024-07-28T19:47:56.208 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~7216589,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,152843223~4376530,188958486~1816264,201778320~5844812,209422996~4801946,216071836~7254506,232482673~3872453,250280585~3186516,271270161~1527390,276135342~3924938,337654823~1447438,367354362~1092722,381707621~5380293,395829986~13298434] 2024-07-28T19:47:56.214 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~7328815,55908035~980737,65261141~6828665,75987387~726387,88080384~2503854,152843223~4376530,188958486~1816264,201326592~21999750,230686720~5668406,247463936~6003165,271270161~8790119,335544320~3557941,367354362~1092722,381681664~5406250,394264576~14863844] 2024-07-28T19:47:56.215 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 6 diff [112226~13187071,16897474~3285069,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,117774156~2460779,152843223~4376530,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,232482673~3872453,250280585~3186516,264356515~4168024,271270161~1527390,276135342~3924938,282017125~1092925,287356099~1845842,320568422~7006352,337654823~1447438,367354362~1092722,381707621~5380293,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.215 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,117774156~2460779,152843223~4376530,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,232482673~3872453,250280585~3186516,264356515~4168024,271270161~1527390,276135342~3924938,282017125~1092925,287356099~1845842,320568422~7006352,337654823~1447438,367354362~1092722,381707621~5380293,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.221 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~3405327,55908035~980737,65261141~6828665,75987387~726387,88080384~2503854,117440512~2794423,152843223~4376530,188958486~1816264,201326592~23856330,230686720~5668406,247463936~6003165,264241152~4283387,271270161~8790119,282017125~1092925,287356099~1845842,319519117~8055657,335544320~3557941,367354362~1092722,381681664~5406250,394264576~14863844,410737048~1385636] 2024-07-28T19:47:56.221 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 7 diff [112226~13187071,16897474~3285069,27561189~3096322,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,117774156~2460779,126138082~3957397,152843223~4376530,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,250280585~3186516,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337654823~1447438,367354362~1092722,381707621~5380293,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.221 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,27561189~3096322,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,117774156~2460779,126138082~3957397,152843223~4376530,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,250280585~3186516,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337654823~1447438,367354362~1092722,381707621~5380293,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.229 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~3405327,27561189~3096322,55908035~980737,65261141~6828665,75987387~726387,88080384~2503854,117440512~2794423,125829120~4266359,152843223~4376530,171966464~6940678,188958486~1816264,201326592~23856330,225682819~1876237,230686720~5668406,247463936~6003165,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,335544320~3557941,367354362~1092722,381681664~5406250,394264576~14863844,410737048~1385636] 2024-07-28T19:47:56.229 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 8 diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,152843223~4376530,166822548~5169376,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337654823~1632958,367354362~1092722,381707621~5380293,388167688~7109454,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.229 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,152843223~4376530,166822548~5169376,174554202~4352940,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337654823~1632958,367354362~1092722,381707621~5380293,388167688~7109454,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.235 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,27561189~8129358,39906052~8672635,50290591~2207245,55908035~980737,65261141~6828665,75987387~726387,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,152843223~4376530,166822548~12084594,188958486~1816264,201326592~23856330,225682819~1876237,230686720~5668406,246603086~7729973,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,335544320~3743461,367354362~1092722,381681664~5406250,388167688~20960732,410737048~1385636] 2024-07-28T19:47:56.235 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 9 diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.235 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,188958486~1816264,201778320~5844812,209422996~4801946,216071836~9111086,225682819~1876237,232482673~3872453,246992343~7340716,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.242 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,27561189~8129358,39906052~8672635,50290591~2207245,52578844~2069890,55908035~980737,65261141~8106725,75987387~726387,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,152843223~4376530,161225466~4529851,166822548~14997648,188958486~1816264,201326592~23856330,225682819~1876237,230686720~5668406,246603086~7729973,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,335544320~6419889,357603777~7571336,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.242 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 10 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~4801946,215999575~9183347,225682819~1876237,231922261~4432865,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.242 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~4801946,215999575~9183347,225682819~1876237,231922261~4432865,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.250 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55908035~980737,65261141~8106725,75987387~726387,83886080~3492882,88080384~2503854,92274688~7640555,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,152843223~4376530,159993769~5761548,166822548~18710419,188958486~1816264,201326592~23856330,225682819~1876237,230686720~5668406,246603086~10522870,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,335544320~6419889,357603777~7571336,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.250 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 11 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~5818018,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,355162073~1084006,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.250 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,188958486~1816264,201778320~5844812,209422996~5818018,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,355162073~1084006,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.258 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75987387~726387,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,152843223~4376530,159993769~5761548,166822548~18710419,188958486~1816264,201326592~23856330,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,335544320~6419889,355162073~1084006,357603777~7571336,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.258 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 12 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,152843223~4376530,159993769~11998155,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.258 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,152843223~4376530,159993769~11998155,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.265 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201326592~23856330,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,335544320~6419889,346870513~3659187,355162073~1084006,357603777~7571336,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.291 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 13 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,152843223~4376530,159993769~12518383,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.291 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,152843223~4376530,159993769~12518383,174554202~10978765,188958486~5054186,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,367354362~1092722,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.293 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~2220464,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201326592~23856330,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,335544320~6419889,346870513~3659187,355162073~1084006,357603777~9110778,367354362~1092722,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.293 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 14 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.293 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,381707621~13569521,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.293 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,38624001~16024733,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,188958486~5054186,197786034~27396888,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,335544320~6419889,346870513~3659187,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.294 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 15 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.294 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,274828887~893301,276135342~5637050,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.294 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,197786034~27396888,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~10502231,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,335544320~6419889,346870513~3659187,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.294 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 16 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,276135342~5637050,282017125~1092925,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.294 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~5844812,207679486~7561528,215999575~9183347,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,276135342~5637050,282017125~1092925,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.294 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,197786034~27396888,225682819~1876237,230686720~8345716,246603086~10522870,260046848~8477691,271270161~10502231,282017125~1092925,287356099~4768251,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.294 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 17 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~9183347,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~7272729,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.294 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~9183347,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~7272729,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.299 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,197786034~27396888,225682819~1876237,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~11839889,287356099~4768251,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.299 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 18 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.299 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.307 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.307 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 19 diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.307 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:56.314 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:47:56.314 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 5 diff [112226~7216589,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,152843223~4376530,209422996~4801946,395829986~13298434] 2024-07-28T19:47:56.314 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~7216589,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,152843223~4376530,209422996~4801946,395829986~13298434] 2024-07-28T19:47:56.321 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~7328815,55908035~980737,65261141~6828665,75987387~726387,88080384~2503854,152843223~4376530,207623132~6601810,394264576~14863844] 2024-07-28T19:47:56.321 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 6 diff [112226~13187071,16897474~3285069,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,117774156~2460779,152843223~4376530,209422996~4801946,220349894~4833028,264356515~4168024,282017125~1092925,287356099~1845842,320568422~7006352,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.321 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,117774156~2460779,152843223~4376530,209422996~4801946,220349894~4833028,264356515~4168024,282017125~1092925,287356099~1845842,320568422~7006352,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.328 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~3405327,55908035~980737,65261141~6828665,75987387~726387,88080384~2503854,117440512~2794423,152843223~4376530,207623132~6601810,220349894~4833028,264241152~4283387,282017125~1092925,287356099~1845842,319519117~8055657,394264576~14863844,410737048~1385636] 2024-07-28T19:47:56.328 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 7 diff [112226~13187071,16897474~3285069,27561189~3096322,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,117774156~2460779,126138082~3957397,152843223~4376530,174554202~4352940,209422996~4801946,220349894~4833028,225682819~1876237,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.328 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,27561189~3096322,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,117774156~2460779,126138082~3957397,152843223~4376530,174554202~4352940,209422996~4801946,220349894~4833028,225682819~1876237,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.335 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~3405327,27561189~3096322,55908035~980737,65261141~6828665,75987387~726387,88080384~2503854,117440512~2794423,125829120~4266359,152843223~4376530,171966464~6940678,207623132~6601810,220349894~4833028,225682819~1876237,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,394264576~14863844,410737048~1385636] 2024-07-28T19:47:56.335 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 8 diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,152843223~4376530,166822548~5169376,174554202~4352940,209422996~4801946,220349894~4833028,225682819~1876237,246992343~7340716,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,338413875~873906,388167688~7109454,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.335 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,55908035~980737,65261141~6828665,75987387~726387,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,152843223~4376530,166822548~5169376,174554202~4352940,209422996~4801946,220349894~4833028,225682819~1876237,246992343~7340716,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,338413875~873906,388167688~7109454,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.342 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,27561189~8129358,39906052~8672635,50290591~2207245,55908035~980737,65261141~6828665,75987387~726387,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,152843223~4376530,166822548~12084594,207623132~6601810,220349894~4833028,225682819~1876237,246603086~7729973,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,338413875~873906,388167688~20960732,410737048~1385636] 2024-07-28T19:47:56.342 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 9 diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,209422996~4801946,217934213~1252500,220349894~4833028,225682819~1876237,246992343~7340716,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.342 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,90327583~256655,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,209422996~4801946,217934213~1252500,220349894~4833028,225682819~1876237,246992343~7340716,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.348 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,27561189~8129358,39906052~8672635,50290591~2207245,52578844~2069890,55908035~980737,65261141~8106725,75987387~726387,88080384~2503854,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,152843223~4376530,161225466~4529851,166822548~14997648,207623132~6601810,217934213~1252500,220349894~4833028,225682819~1876237,246603086~7729973,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,337540878~4423331,357603777~7571336,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.348 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 10 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,209422996~4801946,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~4372786,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.348 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55908035~980737,65261141~8106725,75987387~726387,86185784~1193178,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,209422996~4801946,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~4372786,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.355 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55908035~980737,65261141~8106725,75987387~726387,83886080~3492882,88080384~2503854,92274688~7640555,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,152843223~4376530,159993769~5761548,166822548~18710419,207623132~6601810,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~4372786,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,337540878~4423331,357603777~7571336,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.355 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 11 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,209422996~5818018,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.355 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,152843223~4376530,159993769~5761548,166822548~5169376,174554202~10978765,209422996~5818018,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.363 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75987387~726387,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,152843223~4376530,159993769~5761548,166822548~18710419,207623132~7617882,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,337540878~4423331,355162073~1084006,357603777~7571336,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.363 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 12 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,152843223~4376530,159993769~11998155,174554202~10978765,189908632~4104040,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.363 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,152843223~4376530,159993769~11998155,174554202~10978765,189908632~4104040,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.371 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,149537883~2616756,152843223~4376530,159993769~25539198,189908632~4104040,207623132~7617882,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.371 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 13 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,152843223~4376530,159993769~12518383,174554202~10978765,189908632~4104040,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.371 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,152843223~4376530,159993769~12518383,174554202~10978765,189908632~4104040,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.378 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~2220464,149537883~2616756,152843223~4376530,159993769~25539198,189908632~4104040,207623132~7617882,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.378 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 14 diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,189908632~4104040,201071067~625261,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.378 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~13187071,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,86185784~2649632,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,189908632~4104040,201071067~625261,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,395829986~13298434,410737048~1385636] 2024-07-28T19:47:56.385 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~13299297,16777216~7387956,26493107~9197440,38624001~16024733,55289528~3376295,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159993769~25539198,189908632~4104040,197786034~3910294,207623132~7617882,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.385 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 15 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,201071067~625261,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.386 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,201071067~625261,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.392 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,197786034~3910294,207623132~7617882,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~24097841,410737048~1385636] 2024-07-28T19:47:56.392 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 16 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,201071067~625261,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,279935315~1837077,282017125~1092925,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.392 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,201071067~625261,207679486~7561528,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,279935315~1837077,282017125~1092925,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.398 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~20455478,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~4858702,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,197786034~3910294,207623132~7617882,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,279935315~1837077,282017125~1092925,287356099~4768251,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.399 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 17 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~7272729,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.399 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~7272729,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.406 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~22855959,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~7272729,287356099~4768251,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.406 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 18 diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.406 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~15010906,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,84756138~4620842,90327583~256655,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.413 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~15123132,16777216~7387956,26493107~9197440,38624001~24112034,65261141~8106725,75987387~726387,77036241~6394957,83886080~6698158,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159680442~25852525,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.413 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 19 diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.413 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.420 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:47:56.421 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 6 diff [7209425~6089872,16897474~3285069,117774156~2460779,153891366~718136,220349894~4833028,264356515~4168024,282017125~1092925,287356099~1845842,320568422~7006352,410737048~1385636] 2024-07-28T19:47:56.421 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7209425~6089872,16897474~3285069,117774156~2460779,153891366~718136,220349894~4833028,264356515~4168024,282017125~1092925,287356099~1845842,320568422~7006352,410737048~1385636] 2024-07-28T19:47:56.427 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7209425~6089872,16777216~3405327,117440512~2794423,153891366~718136,220349894~4833028,264241152~4283387,282017125~1092925,287356099~1845842,319519117~8055657,410737048~1385636] 2024-07-28T19:47:56.427 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 7 diff [7209425~6089872,16897474~3285069,27561189~3096322,65561507~2050765,117774156~2460779,126138082~3957397,153891366~718136,174554202~4352940,220349894~4833028,225682819~1876237,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,410737048~1385636] 2024-07-28T19:47:56.427 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7209425~6089872,16897474~3285069,27561189~3096322,65561507~2050765,117774156~2460779,126138082~3957397,153891366~718136,174554202~4352940,220349894~4833028,225682819~1876237,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,410737048~1385636] 2024-07-28T19:47:56.435 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7209425~6089872,16777216~3405327,27561189~3096322,65561507~2050765,117440512~2794423,125829120~4266359,153891366~718136,171966464~6940678,220349894~4833028,225682819~1876237,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,410737048~1385636] 2024-07-28T19:47:56.435 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 8 diff [7209425~6089872,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,65561507~2050765,109479491~5347846,117774156~2460779,126138082~3957397,153891366~718136,166822548~5169376,174554202~4352940,220349894~4833028,225682819~1876237,246992343~7340716,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,338413875~873906,388167688~7109454,410737048~1385636] 2024-07-28T19:47:56.436 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7209425~6089872,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,65561507~2050765,109479491~5347846,117774156~2460779,126138082~3957397,153891366~718136,166822548~5169376,174554202~4352940,220349894~4833028,225682819~1876237,246992343~7340716,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,338413875~873906,388167688~7109454,410737048~1385636] 2024-07-28T19:47:56.441 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7209425~6089872,16777216~7387956,27561189~8129358,39906052~8672635,50290591~2207245,65561507~2050765,109051904~5775433,117440512~2794423,125829120~4266359,153891366~718136,166822548~12084594,220349894~4833028,225682819~1876237,246603086~7729973,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,338413875~873906,388167688~7109454,410737048~1385636] 2024-07-28T19:47:56.441 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 9 diff [7209425~6089872,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,54264737~383997,65561507~2050765,71835562~1532304,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,153891366~718136,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,211982304~909576,217934213~1252500,220349894~4833028,225682819~1876237,246992343~7340716,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.441 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7209425~6089872,16897474~3285069,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,54264737~383997,65561507~2050765,71835562~1532304,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,153891366~718136,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,211982304~909576,217934213~1252500,220349894~4833028,225682819~1876237,246992343~7340716,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.447 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7209425~6089872,16777216~7387956,27561189~8129358,39906052~8672635,50290591~2207245,52578844~2069890,65561507~2050765,71835562~1532304,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,153891366~718136,161225466~4529851,166822548~14997648,211982304~909576,217934213~1252500,220349894~4833028,225682819~1876237,246603086~7729973,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,337540878~4423331,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.447 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 10 diff [7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,65561507~2050765,71835562~1532304,86185784~1193178,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,153891366~718136,159993769~5761548,166822548~5169376,174554202~10978765,211982304~909576,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~4372786,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.448 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,65561507~2050765,71835562~1532304,86185784~1193178,92287280~7627963,109479491~5347846,117774156~2460779,126138082~3957397,139049388~2220464,153891366~718136,159993769~5761548,166822548~5169376,174554202~10978765,211982304~909576,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~4372786,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.454 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7209425~6089872,16777216~7387956,26493107~9197440,39906052~14742682,65561507~2050765,71835562~1532304,83886080~3492882,92274688~7640555,109051904~5775433,117440512~2794423,125829120~4266359,139049388~2220464,153891366~718136,159993769~5761548,166822548~18710419,211982304~909576,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~4372786,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,337540878~4423331,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.454 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 11 diff [7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,86185784~2649632,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,153891366~718136,159993769~5761548,166822548~5169376,174554202~10978765,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.454 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,86185784~2649632,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,153891366~718136,159993769~5761548,166822548~5169376,174554202~10978765,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308806914~7607545,320568422~7006352,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.461 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7209425~6089872,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,83886080~4949336,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,153891366~718136,159993769~5761548,166822548~18710419,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,308516915~7897544,319519117~8055657,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.462 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 12 diff [3636392~2148951,7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,153891366~718136,159993769~11998155,174554202~10978765,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.462 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,109479491~5347846,117774156~2460779,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,153891366~718136,159993769~11998155,174554202~10978765,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.467 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7209425~6089872,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,109051904~5775433,117440512~7406213,125829120~4266359,139049388~2220464,149537883~2616756,153891366~718136,159993769~25539198,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.468 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 13 diff [3636392~2148951,7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,153891366~718136,159993769~12518383,174554202~10978765,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.468 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,153891366~718136,159993769~12518383,174554202~10978765,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.474 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7209425~6089872,16777216~7387956,26493107~9197440,39906052~14742682,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,104857600~19989125,125829120~8819527,139049388~2220464,149537883~2616756,153891366~718136,159993769~25539198,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563,410737048~1385636] 2024-07-28T19:47:56.474 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 14 diff [3636392~2148951,7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,153891366~718136,159993769~25539198,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463,410737048~1385636] 2024-07-28T19:47:56.474 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7209425~6089872,16897474~3285069,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,153891366~718136,159993769~25539198,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463,410737048~1385636] 2024-07-28T19:47:56.481 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7209425~6089872,16777216~7387956,26493107~9197440,38624001~16024733,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,153891366~718136,159993769~25539198,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463,410737048~1385636] 2024-07-28T19:47:56.481 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 15 diff [992284~14130848,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942,410737048~1385636] 2024-07-28T19:47:56.481 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,320568422~7006352,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942,410737048~1385636] 2024-07-28T19:47:56.488 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,16777216~7387956,26493107~9197440,38624001~20455478,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,83886080~5490900,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,274828887~893301,279935315~1837077,282017125~1092925,287356099~1845842,304100381~12314078,319519117~8055657,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942,410737048~1385636] 2024-07-28T19:47:56.488 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 16 diff [992284~14130848,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,279935315~1837077,282017125~1092925,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.488 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,16897474~3285069,20365656~3799516,26493107~9197440,38624001~20455478,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,279935315~1837077,282017125~1092925,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.495 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,16777216~7387956,26493107~9197440,38624001~20455478,65561507~2050765,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,125829120~8819527,139049388~4858702,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,279935315~1837077,282017125~1092925,287356099~4768251,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.495 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 17 diff [992284~14130848,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,201071067~625261,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~7272729,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.495 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,16897474~3285069,20365656~3799516,26493107~9197440,38624001~22855959,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,201071067~625261,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~7272729,287356099~4768251,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.502 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,16777216~7387956,26493107~9197440,38624001~22855959,65561507~2050765,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,197786034~3910294,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,220349894~4833028,225682819~1876237,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~7272729,287356099~4768251,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.502 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 18 diff [992284~14130848,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.502 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,16897474~3285069,20365656~3799516,26493107~9197440,38624001~24112034,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.509 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,16777216~7387956,26493107~9197440,38624001~24112034,65561507~2050765,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159680442~25852525,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.509 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 19 diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.509 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.516 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:47:56.516 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 7 diff [27561189~3096322,65561507~2050765,126138082~3957397,174554202~4352940,225682819~1876237,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545] 2024-07-28T19:47:56.516 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [27561189~3096322,65561507~2050765,126138082~3957397,174554202~4352940,225682819~1876237,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545] 2024-07-28T19:47:56.524 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [27561189~3096322,65561507~2050765,125829120~4266359,171966464~6940678,225682819~1876237,260046848~5233035,274828887~893301,279935315~1837077,308516915~7897544] 2024-07-28T19:47:56.524 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 8 diff [7481273~2119565,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,65561507~2050765,109479491~5347846,126138082~3957397,166822548~5169376,174554202~4352940,225682819~1876237,246992343~7340716,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545,338413875~873906,388167688~7109454] 2024-07-28T19:47:56.524 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7481273~2119565,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,65561507~2050765,109479491~5347846,126138082~3957397,166822548~5169376,174554202~4352940,225682819~1876237,246992343~7340716,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545,338413875~873906,388167688~7109454] 2024-07-28T19:47:56.531 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7481273~2119565,20182543~3982629,27561189~8129358,39906052~8672635,50290591~2207245,65561507~2050765,109051904~5775433,125829120~4266359,166822548~12084594,225682819~1876237,246603086~7729973,260046848~5233035,274828887~893301,279935315~1837077,308516915~7897544,338413875~873906,388167688~7109454] 2024-07-28T19:47:56.531 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 9 diff [7481273~2119565,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,54264737~383997,65561507~2050765,71835562~1532304,109479491~5347846,126138082~3957397,139049388~2220464,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,211982304~909576,217934213~1252500,225682819~1876237,246992343~7340716,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.531 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7481273~2119565,20365656~3799516,27561189~8129358,41510874~7067813,50290591~2207245,54264737~383997,65561507~2050765,71835562~1532304,109479491~5347846,126138082~3957397,139049388~2220464,161225466~4529851,166822548~5169376,174554202~4352940,179002214~2817982,211982304~909576,217934213~1252500,225682819~1876237,246992343~7340716,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.537 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7481273~2119565,20182543~3982629,27561189~8129358,39906052~8672635,50290591~2207245,52578844~2069890,65561507~2050765,71835562~1532304,109051904~5775433,125829120~4266359,139049388~2220464,161225466~4529851,166822548~14997648,211982304~909576,217934213~1252500,225682819~1876237,246603086~7729973,260046848~5233035,274828887~893301,279935315~1837077,308516915~7897544,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.537 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 10 diff [7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,65561507~2050765,71835562~1532304,86185784~1193178,92287280~7627963,109479491~5347846,126138082~3957397,139049388~2220464,159993769~5761548,166822548~5169376,174554202~10978765,211982304~909576,215999575~1776727,217934213~1252500,225682819~1876237,231922261~4372786,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.537 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,65561507~2050765,71835562~1532304,86185784~1193178,92287280~7627963,109479491~5347846,126138082~3957397,139049388~2220464,159993769~5761548,166822548~5169376,174554202~10978765,211982304~909576,215999575~1776727,217934213~1252500,225682819~1876237,231922261~4372786,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.544 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7481273~2119565,20182543~3982629,26493107~9197440,39906052~14742682,65561507~2050765,71835562~1532304,83886080~3492882,92274688~7640555,109051904~5775433,125829120~4266359,139049388~2220464,159993769~5761548,166822548~18710419,211982304~909576,215999575~1776727,217934213~1252500,225682819~1876237,231922261~4372786,246603086~10522870,260046848~5233035,274828887~893301,279935315~1837077,308516915~7897544,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.544 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 11 diff [7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,86185784~2649632,92287280~7627963,109479491~5347846,120515010~4331715,126138082~3957397,139049388~2220464,159993769~5761548,166822548~5169376,174554202~10978765,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.544 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,86185784~2649632,92287280~7627963,109479491~5347846,120515010~4331715,126138082~3957397,139049388~2220464,159993769~5761548,166822548~5169376,174554202~10978765,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,308806914~7607545,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.550 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7481273~2119565,20182543~3982629,26493107~9197440,39906052~14742682,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,83886080~4949336,92274688~7640555,109051904~5775433,120234935~4611790,125829120~4266359,139049388~2220464,159993769~5761548,166822548~18710419,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246603086~10522870,260046848~5233035,274828887~893301,279935315~1837077,308516915~7897544,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.550 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 12 diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,109479491~5347846,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,159993769~11998155,174554202~10978765,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,304100381~12314078,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.550 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,109479491~5347846,120515010~4331715,126138082~3957397,139049388~2220464,149537883~2616756,159993769~11998155,174554202~10978765,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,304100381~12314078,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.557 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7481273~2119565,20182543~3982629,26493107~9197440,39906052~14742682,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,109051904~5775433,120234935~4611790,125829120~4266359,139049388~2220464,149537883~2616756,159993769~25539198,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246603086~10522870,260046848~5233035,274828887~893301,279935315~1837077,304100381~12314078,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.557 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 13 diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,159993769~12518383,174554202~10978765,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563] 2024-07-28T19:47:56.557 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108462095~16384630,126138082~8510565,139049388~2220464,149537883~2616756,159993769~12518383,174554202~10978765,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563] 2024-07-28T19:47:56.564 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7481273~2119565,20182543~3982629,26493107~9197440,39906052~14742682,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,104857600~19989125,125829120~8819527,139049388~2220464,149537883~2616756,159993769~25539198,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246603086~10522870,260046848~5233035,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563] 2024-07-28T19:47:56.564 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 14 diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,159993769~25539198,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463] 2024-07-28T19:47:56.564 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,159993769~25539198,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463] 2024-07-28T19:47:56.570 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7481273~2119565,20182543~3982629,26493107~9197440,38624001~16024733,55289528~3376295,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,159993769~25539198,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246603086~10522870,260046848~5233035,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463] 2024-07-28T19:47:56.570 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 15 diff [992284~8608554,11026132~4097000,20365656~3799516,26493107~9197440,38624001~20455478,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.571 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~8608554,11026132~4097000,20365656~3799516,26493107~9197440,38624001~20455478,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.577 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~8608554,11026132~4097000,20182543~3982629,26493107~9197440,38624001~20455478,65561507~2050765,70261440~651495,71835562~1532304,77036241~6394957,83886080~5490900,92274688~7640555,104857600~19989125,125829120~8819527,139049388~4858702,149537883~2616756,159680442~25852525,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246603086~10522870,260046848~5233035,274828887~893301,279935315~1837077,304100381~12314078,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.577 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 16 diff [992284~8608554,11026132~4097000,20365656~3799516,26493107~9197440,38624001~20455478,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,279935315~1837077,288553325~3571025,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.577 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~8608554,11026132~4097000,20365656~3799516,26493107~9197440,38624001~20455478,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~4858702,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,279935315~1837077,288553325~3571025,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.584 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~8608554,11026132~4097000,20182543~3982629,26493107~9197440,38624001~20455478,65561507~2050765,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,125829120~8819527,139049388~4858702,149537883~2616756,159680442~25852525,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,279935315~1837077,288553325~3571025,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.584 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 17 diff [992284~14130848,20365656~3799516,26493107~9197440,38624001~22855959,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~6859657,288553325~3571025,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.584 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20365656~3799516,26493107~9197440,38624001~22855959,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~6859657,288553325~3571025,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.590 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20182543~3982629,26493107~9197440,38624001~22855959,65561507~2050765,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,197786034~3910294,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,225682819~1876237,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~6859657,288553325~3571025,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.590 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 18 diff [992284~14130848,20365656~3799516,26493107~9197440,38624001~24112034,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.590 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20365656~3799516,26493107~9197440,38624001~24112034,65561507~2050765,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.596 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20182543~3982629,26493107~9197440,38624001~24112034,65561507~2050765,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.596 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 19 diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:56.597 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:56.603 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:56.603 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 8 diff [7481273~2119565,20365656~3799516,27981601~7708946,41510874~7067813,50290591~2207245,109479491~5347846,166822548~5169376,246992343~7340716,338413875~873906,388167688~7109454] 2024-07-28T19:47:56.603 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7481273~2119565,20365656~3799516,27981601~7708946,41510874~7067813,50290591~2207245,109479491~5347846,166822548~5169376,246992343~7340716,338413875~873906,388167688~7109454] 2024-07-28T19:47:56.610 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7481273~2119565,20182543~3982629,27981601~7708946,39906052~8672635,50290591~2207245,109051904~5775433,166822548~5169376,246603086~7729973,338413875~873906,388167688~7109454] 2024-07-28T19:47:56.610 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 9 diff [7481273~2119565,20365656~3799516,27981601~7708946,41510874~7067813,50290591~2207245,54264737~383997,71835562~1532304,109479491~5347846,139049388~2220464,161225466~4529851,166822548~5169376,179002214~2817982,211982304~909576,217934213~1252500,246992343~7340716,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.610 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7481273~2119565,20365656~3799516,27981601~7708946,41510874~7067813,50290591~2207245,54264737~383997,71835562~1532304,109479491~5347846,139049388~2220464,161225466~4529851,166822548~5169376,179002214~2817982,211982304~909576,217934213~1252500,246992343~7340716,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.617 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7481273~2119565,20182543~3982629,27981601~7708946,39906052~8672635,50290591~2207245,52578844~2069890,71835562~1532304,109051904~5775433,139049388~2220464,161225466~4529851,166822548~5169376,178907142~2913054,211982304~909576,217934213~1252500,246603086~7729973,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.617 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 10 diff [7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,71835562~1532304,86185784~1193178,92287280~7627963,109479491~5347846,139049388~2220464,159993769~5761548,166822548~5169376,177959428~7573539,211982304~909576,215999575~1776727,217934213~1252500,231922261~4372786,246992343~10133613,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.617 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,71835562~1532304,86185784~1193178,92287280~7627963,109479491~5347846,139049388~2220464,159993769~5761548,166822548~5169376,177959428~7573539,211982304~909576,215999575~1776727,217934213~1252500,231922261~4372786,246992343~10133613,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.624 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7481273~2119565,20182543~3982629,26493107~9197440,39906052~14742682,71835562~1532304,83886080~3492882,92274688~7640555,109051904~5775433,139049388~2220464,159993769~5761548,166822548~5169376,177959428~7573539,211982304~909576,215999575~1776727,217934213~1252500,231922261~4372786,246603086~10522870,337540878~4423331,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.624 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 11 diff [7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,86185784~2649632,92287280~7627963,109479491~5347846,120515010~4331715,139049388~2220464,159993769~5761548,166822548~5169376,177959428~7573539,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.624 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,86185784~2649632,92287280~7627963,109479491~5347846,120515010~4331715,139049388~2220464,159993769~5761548,166822548~5169376,177959428~7573539,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.630 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [7481273~2119565,20182543~3982629,26493107~9197440,39906052~14742682,55289528~3376295,70261440~651495,71835562~1532304,83886080~4949336,92274688~7640555,109051904~5775433,120234935~4611790,139049388~2220464,159993769~5761548,166822548~5169376,177959428~7573539,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246603086~10522870,337540878~4423331,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.630 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 12 diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,109479491~5347846,120515010~4331715,139049388~2220464,149537883~2616756,159993769~11998155,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,304100381~6208439,311478804~409659,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.631 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,109479491~5347846,120515010~4331715,139049388~2220464,149537883~2616756,159993769~11998155,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,304100381~6208439,311478804~409659,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.637 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7481273~2119565,20182543~3982629,26493107~9197440,39906052~14742682,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,109051904~5775433,120234935~4611790,139049388~2220464,149537883~2616756,159993769~11998155,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246603086~10522870,304100381~6208439,311478804~409659,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~10246563] 2024-07-28T19:47:56.637 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 13 diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108462095~16384630,127746465~6902182,139049388~2220464,149537883~2616756,159993769~12518383,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563] 2024-07-28T19:47:56.638 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,41510874~12237116,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108462095~16384630,127746465~6902182,139049388~2220464,149537883~2616756,159993769~12518383,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563] 2024-07-28T19:47:56.644 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7481273~2119565,20182543~3982629,26493107~9197440,39906052~14742682,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,104857600~19989125,127746465~6902182,139049388~2220464,149537883~2616756,159993769~12518383,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246603086~10522870,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~10246563] 2024-07-28T19:47:56.644 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 14 diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159993769~14927216,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463] 2024-07-28T19:47:56.644 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,7481273~2119565,20365656~3799516,26493107~9197440,38624001~15123989,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159993769~14927216,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463] 2024-07-28T19:47:56.650 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,7481273~2119565,20182543~3982629,26493107~9197440,38624001~16024733,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,104857600~19989125,127746465~6902182,139049388~4858702,149537883~2616756,159993769~14927216,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246603086~10522870,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~10246563,399392058~7026463] 2024-07-28T19:47:56.651 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 15 diff [992284~8608554,11026132~4097000,20365656~3799516,26493107~9197440,38624001~20455478,70261440~651495,71835562~1532304,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.651 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~8608554,11026132~4097000,20365656~3799516,26493107~9197440,38624001~20455478,70261440~651495,71835562~1532304,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.657 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~8608554,11026132~4097000,20182543~3982629,26493107~9197440,38624001~20455478,70261440~651495,71835562~1532304,77036241~6394957,83886080~5490900,92274688~7640555,104857600~19989125,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246603086~10522870,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.657 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 16 diff [992284~8608554,11026132~4097000,20365656~3799516,26493107~9197440,38624001~20455478,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.657 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~8608554,11026132~4097000,20365656~3799516,26493107~9197440,38624001~20455478,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246992343~10133613,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.663 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~8608554,11026132~4097000,20182543~3982629,26493107~9197440,38624001~20455478,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,246603086~10522870,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.663 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 17 diff [992284~14130848,20365656~3799516,26493107~9197440,38624001~22855959,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.663 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20365656~3799516,26493107~9197440,38624001~22855959,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.670 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20182543~3982629,26493107~9197440,38624001~22855959,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.670 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 18 diff [992284~14130848,20365656~3799516,26493107~9197440,38624001~24112034,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.670 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20365656~3799516,26493107~9197440,38624001~24112034,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.678 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20182543~3982629,26493107~9197440,38624001~24112034,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.678 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 19 diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:56.678 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:56.683 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:56.683 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 9 diff [54264737~383997,71835562~1532304,139049388~2220464,161225466~4529851,179002214~2817982,211982304~909576,217934213~1252500,337540878~4423331,357603777~7571336,385030579~8097932] 2024-07-28T19:47:56.683 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [54264737~383997,71835562~1532304,139049388~2220464,161225466~4529851,179002214~2817982,211982304~909576,217934213~1252500,337540878~4423331,357603777~7571336,385030579~8097932] 2024-07-28T19:47:56.689 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [52578844~2069890,71835562~1532304,139049388~2220464,161225466~4529851,178907142~2913054,211982304~909576,217934213~1252500,337540878~4423331,357603777~7571336,385030579~8097932] 2024-07-28T19:47:56.689 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 10 diff [26493107~2633969,45794658~7953332,54264737~383997,71835562~1532304,86185784~1193178,92287280~7627963,139049388~2220464,159993769~5761548,177959428~7573539,211982304~909576,215999575~1776727,217934213~1252500,231922261~4372786,253887767~3238189,337540878~4423331,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.689 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [26493107~2633969,45794658~7953332,54264737~383997,71835562~1532304,86185784~1193178,92287280~7627963,139049388~2220464,159993769~5761548,177959428~7573539,211982304~909576,215999575~1776727,217934213~1252500,231922261~4372786,253887767~3238189,337540878~4423331,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.695 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [26493107~2633969,45794658~8854076,71835562~1532304,83886080~3492882,92274688~7640555,139049388~2220464,159993769~5761548,177959428~7573539,211982304~909576,215999575~1776727,217934213~1252500,231922261~4372786,253887767~3238189,337540878~4423331,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.695 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 11 diff [20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,86185784~2649632,92287280~7627963,120515010~4331715,139049388~2220464,159993769~5761548,177959428~7573539,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,337540878~4423331,355162073~1084006,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.695 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,86185784~2649632,92287280~7627963,120515010~4331715,139049388~2220464,159993769~5761548,177959428~7573539,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,337540878~4423331,355162073~1084006,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.701 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [20780229~2485981,26493107~2633969,32790789~1112582,45794658~8854076,55289528~3376295,70261440~651495,71835562~1532304,83886080~4949336,92274688~7640555,120234935~4611790,139049388~2220464,159993769~5761548,177959428~7573539,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,337540878~4423331,355162073~1084006,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.701 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 12 diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,120515010~4331715,139049388~2220464,149537883~2616756,159993769~9292655,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.701 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,120515010~4331715,139049388~2220464,149537883~2616756,159993769~9292655,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.707 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~8854076,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,120234935~4611790,139049388~2220464,149537883~2616756,159993769~9292655,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,337540878~4423331,346870513~3659187,355162073~1084006,357603777~7571336,385030579~8911934] 2024-07-28T19:47:56.708 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 13 diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108462095~3696521,113389514~11457211,127746465~6902182,139049388~2220464,149537883~2616756,159993769~12518383,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~8911934] 2024-07-28T19:47:56.708 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108462095~3696521,113389514~11457211,127746465~6902182,139049388~2220464,149537883~2616756,159993769~12518383,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~8911934] 2024-07-28T19:47:56.714 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~8854076,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,104857600~7301016,113389514~11457211,127746465~6902182,139049388~2220464,149537883~2616756,159993769~12518383,176051091~9481876,189908632~4104040,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~9110778,385030579~8911934] 2024-07-28T19:47:56.714 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 14 diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,38624001~3004469,45794658~7953332,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159993769~14927216,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~8911934,399392058~7026463] 2024-07-28T19:47:56.714 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,38624001~3004469,45794658~7953332,54264737~383997,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159993769~14927216,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~8911934,399392058~7026463] 2024-07-28T19:47:56.721 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,38624001~3004469,45794658~8854076,55289528~3376295,70261440~651495,71835562~1532304,77036241~6394957,83886080~4949336,92274688~7640555,104857600~19989125,127746465~6902182,139049388~4858702,149537883~2616756,159993769~14927216,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~8911934,399392058~7026463] 2024-07-28T19:47:56.722 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 15 diff [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,70261440~651495,71835562~1532304,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.722 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,70261440~651495,71835562~1532304,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.728 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,70261440~651495,71835562~1532304,77036241~6394957,83886080~5490900,92274688~7640555,104857600~19989125,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,337540878~4423331,346870513~3659187,355162073~1084006,357603777~13773811,385030579~21387942] 2024-07-28T19:47:56.728 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 16 diff [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.728 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.734 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139049388~4858702,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,253887767~3238189,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.734 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 17 diff [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~15685302,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.734 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~15685302,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.743 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~15685302,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,205246723~5322942,211982304~909576,213375755~1865259,215999575~1776727,217934213~1252500,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.743 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 18 diff [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,45794658~16941377,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.743 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,45794658~16941377,67707114~5660752,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.751 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,45794658~16941377,67707114~5660752,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159680442~25852525,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,383067522~24933921] 2024-07-28T19:47:56.751 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 19 diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:56.751 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:56.757 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:56.758 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 10 diff [26493107~2633969,45794658~7953332,86185784~1193178,92287280~7627963,159993769~1588339,177959428~7573539,215999575~1776727,231922261~4372786,253887767~3238189,391211106~2731407] 2024-07-28T19:47:56.758 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [26493107~2633969,45794658~7953332,86185784~1193178,92287280~7627963,159993769~1588339,177959428~7573539,215999575~1776727,231922261~4372786,253887767~3238189,391211106~2731407] 2024-07-28T19:47:56.764 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [26493107~2633969,45794658~7953332,83886080~3492882,92274688~7640555,159993769~1588339,177959428~7573539,215999575~1776727,231922261~4372786,253887767~3238189,391211106~2731407] 2024-07-28T19:47:56.764 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 11 diff [20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,86185784~2649632,92287280~7627963,120515010~4331715,159993769~1588339,177959428~7573539,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,355162073~1084006,391211106~2731407] 2024-07-28T19:47:56.764 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,86185784~2649632,92287280~7627963,120515010~4331715,159993769~1588339,177959428~7573539,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,355162073~1084006,391211106~2731407] 2024-07-28T19:47:56.771 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,83886080~4949336,92274688~7640555,120234935~4611790,159993769~1588339,177959428~7573539,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,355162073~1084006,391211106~2731407] 2024-07-28T19:47:56.771 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 12 diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,86185784~2649632,92287280~7627963,120515010~4331715,149537883~2616756,159993769~1588339,163865563~5420861,176051091~9481876,189908632~4104040,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,346870513~3659187,355162073~1084006,391211106~2731407] 2024-07-28T19:47:56.771 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,86185784~2649632,92287280~7627963,120515010~4331715,149537883~2616756,159993769~1588339,163865563~5420861,176051091~9481876,189908632~4104040,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,346870513~3659187,355162073~1084006,391211106~2731407] 2024-07-28T19:47:56.777 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,83886080~4949336,92274688~7640555,120234935~4611790,149537883~2616756,159993769~1588339,163865563~5420861,176051091~9481876,189908632~4104040,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,346870513~3659187,355162073~1084006,391211106~2731407] 2024-07-28T19:47:56.778 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 13 diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,86185784~2649632,92287280~7627963,108462095~3696521,113389514~11457211,127746465~6902182,149537883~2616756,159993769~1588339,162125061~10387091,176051091~9481876,189908632~4104040,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,364820588~1893967,391211106~2731407] 2024-07-28T19:47:56.778 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,86185784~2649632,92287280~7627963,108462095~3696521,113389514~11457211,127746465~6902182,149537883~2616756,159993769~1588339,162125061~10387091,176051091~9481876,189908632~4104040,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,364820588~1893967,391211106~2731407] 2024-07-28T19:47:56.784 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,83886080~4949336,92274688~7640555,104857600~7301016,113389514~11457211,127746465~6902182,149537883~2616756,159993769~1588339,162125061~10387091,176051091~9481876,189908632~4104040,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,364820588~1893967,391211106~2731407] 2024-07-28T19:47:56.784 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 14 diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,38624001~3004469,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159993769~1588339,162125061~12795924,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,363707150~7670438,391211106~2731407,399392058~7026463] 2024-07-28T19:47:56.784 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,38624001~3004469,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,86185784~2649632,92287280~7627963,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159993769~1588339,162125061~12795924,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,363707150~7670438,391211106~2731407,399392058~7026463] 2024-07-28T19:47:56.790 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,26493107~2633969,32790789~1112582,38624001~3004469,45794658~7953332,55289528~3376295,70261440~651495,77036241~6394957,83886080~4949336,92274688~7640555,104857600~19989125,127746465~6902182,139059271~4848819,149537883~2616756,159993769~1588339,162125061~12795924,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,363707150~7670438,391211106~2731407,399392058~7026463] 2024-07-28T19:47:56.790 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 15 diff [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,70261440~651495,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,360999874~2416421,363707150~7670438,391211106~15207415] 2024-07-28T19:47:56.790 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,70261440~651495,77036241~6394957,84756138~4620842,92287280~7627963,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,360999874~2416421,363707150~7670438,391211106~15207415] 2024-07-28T19:47:56.797 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,70261440~651495,77036241~6394957,83886080~5490900,92274688~7640555,104857600~19989125,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,360999874~2416421,363707150~7670438,391211106~15207415] 2024-07-28T19:47:56.797 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 16 diff [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,67707114~4813819,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.797 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,67707114~4813819,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.804 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~13284821,67707114~4813819,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,207679486~2890179,213375755~1865259,215999575~1776727,231922261~7110175,253887767~3238189,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.804 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 17 diff [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~15685302,67707114~4813819,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,205246723~5322942,213375755~1865259,215999575~1776727,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.804 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~15685302,67707114~4813819,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,201071067~625261,205246723~5322942,213375755~1865259,215999575~1776727,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.810 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,45794658~15685302,67707114~4813819,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~9481876,189908632~4104040,197786034~3910294,205246723~5322942,213375755~1865259,215999575~1776727,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.810 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 18 diff [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,45794658~16941377,67707114~4813819,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.810 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,45794658~16941377,67707114~4813819,77036241~6394957,84756138~4620842,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159680442~25852525,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.816 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,45794658~16941377,67707114~4813819,77036241~6394957,83886080~5490900,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159680442~25852525,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.816 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 19 diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.816 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.822 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:47:56.822 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 11 diff [20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,87027871~1807545,120515010~4331715,213375755~1865259,235936831~3095605,256606700~476669,355162073~1084006] 2024-07-28T19:47:56.822 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,87027871~1807545,120515010~4331715,213375755~1865259,235936831~3095605,256606700~476669,355162073~1084006] 2024-07-28T19:47:56.828 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,87027871~1807545,120234935~4611790,213375755~1865259,235936831~3095605,256606700~476669,355162073~1084006] 2024-07-28T19:47:56.828 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 12 diff [3636392~2148951,20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,120515010~4331715,149537883~2616756,163865563~5420861,176051091~3643688,189908632~4104040,207679486~2890179,213375755~1865259,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,346870513~3659187,355162073~1084006] 2024-07-28T19:47:56.828 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,120515010~4331715,149537883~2616756,163865563~5420861,176051091~3643688,189908632~4104040,207679486~2890179,213375755~1865259,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,346870513~3659187,355162073~1084006] 2024-07-28T19:47:56.835 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,120234935~4611790,149537883~2616756,163865563~5420861,176051091~3643688,189908632~4104040,207679486~2890179,213375755~1865259,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,346870513~3659187,355162073~1084006] 2024-07-28T19:47:56.835 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 13 diff [3636392~2148951,20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,108462095~3696521,113389514~11457211,127746465~6902182,149537883~2616756,162125061~10387091,176051091~3643688,189908632~4104040,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,364820588~1893967] 2024-07-28T19:47:56.835 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,108462095~3696521,113389514~11457211,127746465~6902182,149537883~2616756,162125061~10387091,176051091~3643688,189908632~4104040,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,364820588~1893967] 2024-07-28T19:47:56.841 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,32790789~1112582,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,104857600~7301016,113389514~11457211,127746465~6902182,149537883~2616756,162125061~10387091,176051091~3643688,189908632~4104040,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,364820588~1893967] 2024-07-28T19:47:56.841 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 14 diff [3636392~2148951,20780229~2485981,32790789~1112582,38624001~3004469,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,98333376~1495812,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,162125061~12795924,176051091~3643688,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.841 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,20780229~2485981,32790789~1112582,38624001~3004469,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,98333376~1495812,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,162125061~12795924,176051091~3643688,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.847 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,20780229~2485981,32790789~1112582,38624001~3004469,55289528~3376295,70261440~651495,77036241~6394957,87027871~1807545,98333376~1495812,104857600~19989125,127746465~6902182,139059271~4848819,149537883~2616756,162125061~12795924,176051091~3643688,189908632~4104040,197786034~3910294,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.847 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 15 diff [992284~7616291,11026132~4097000,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53284707~5794772,70261440~651495,77036241~6394957,84756138~4620842,98333376~1495812,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~3643688,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.847 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53284707~5794772,70261440~651495,77036241~6394957,84756138~4620842,98333376~1495812,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~3643688,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.854 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53284707~5794772,70261440~651495,77036241~6394957,84756138~4620842,98333376~1495812,104857600~19989125,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~3643688,189908632~4104040,197786034~3910294,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,355162073~1084006,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.854 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 16 diff [992284~7616291,11026132~4097000,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53284707~5794772,67707114~4813819,77036241~6394957,84756138~4620842,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.854 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53284707~5794772,67707114~4813819,77036241~6394957,84756138~4620842,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,201071067~625261,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.860 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53284707~5794772,67707114~4813819,77036241~6394957,84756138~4620842,98333376~1495812,100663296~24183429,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,197786034~3910294,207679486~2890179,213375755~1865259,233701664~1501648,235936831~3095605,256606700~476669,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.860 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 17 diff [992284~14130848,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53164013~8315947,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,201071067~625261,205246723~5322942,213375755~1865259,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.860 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53164013~8315947,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,201071067~625261,205246723~5322942,213375755~1865259,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.867 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,53164013~8315947,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,197786034~3910294,205246723~5322942,213375755~1865259,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.867 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 18 diff [992284~14130848,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159680442~22266461,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.867 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159680442~22266461,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.873 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,20780229~2485981,29732490~296310,32790789~1112582,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159680442~22266461,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.873 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 19 diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.873 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.879 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.879 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 12 diff [3636392~2148951,77036241~6394957,149537883~2616756,163865563~5420861,176051091~3643688,189908632~4104040,207679486~2890179,304100381~6208439,311478804~409659,346870513~3659187] 2024-07-28T19:47:56.879 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,77036241~6394957,149537883~2616756,163865563~5420861,176051091~3643688,189908632~4104040,207679486~2890179,304100381~6208439,311478804~409659,346870513~3659187] 2024-07-28T19:47:56.885 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,77036241~6394957,149537883~2616756,163865563~5420861,176051091~3643688,189908632~4104040,207679486~2890179,304100381~6208439,311478804~409659,346870513~3659187] 2024-07-28T19:47:56.885 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 13 diff [3636392~2148951,77036241~6394957,108462095~3696521,113389514~7808033,127746465~6902182,149537883~2616756,162125061~10387091,176051091~3643688,189908632~4104040,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,364820588~1893967] 2024-07-28T19:47:56.885 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,77036241~6394957,108462095~3696521,113389514~7808033,127746465~6902182,149537883~2616756,162125061~10387091,176051091~3643688,189908632~4104040,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,364820588~1893967] 2024-07-28T19:47:56.891 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,77036241~6394957,104857600~7301016,113389514~7808033,127746465~6902182,149537883~2616756,162125061~10387091,176051091~3643688,189908632~4104040,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,364820588~1893967] 2024-07-28T19:47:56.891 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 14 diff [3636392~2148951,38624001~3004469,77036241~6394957,98333376~1495812,108254853~12942694,127746465~6902182,139059271~4848819,149537883~2616756,162125061~12795924,176051091~3643688,189908632~4104040,201071067~625261,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.891 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [3636392~2148951,38624001~3004469,77036241~6394957,98333376~1495812,108254853~12942694,127746465~6902182,139059271~4848819,149537883~2616756,162125061~12795924,176051091~3643688,189908632~4104040,201071067~625261,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.897 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [3636392~2148951,38624001~3004469,77036241~6394957,98333376~1495812,104857600~16339947,127746465~6902182,139059271~4848819,149537883~2616756,162125061~12795924,176051091~3643688,189908632~4104040,197786034~3910294,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.897 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 15 diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,77036241~6394957,84756138~4620842,98333376~1495812,108254853~12942694,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~3643688,189908632~4104040,201071067~625261,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.897 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,77036241~6394957,84756138~4620842,98333376~1495812,108254853~12942694,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~3643688,189908632~4104040,201071067~625261,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.904 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,77036241~6394957,84756138~4620842,98333376~1495812,104857600~16339947,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~3643688,189908632~4104040,197786034~3910294,207679486~2890179,233701664~1501648,304100381~6208439,311478804~409659,330570804~2468514,346870513~3659187,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.904 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 16 diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,77036241~6394957,84756138~4620842,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,201071067~625261,207679486~2890179,233701664~1501648,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.904 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,77036241~6394957,84756138~4620842,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,201071067~625261,207679486~2890179,233701664~1501648,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.910 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,77036241~6394957,84756138~4620842,98333376~1495812,100663296~20534251,127746465~6902182,139059271~4848819,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,197786034~3910294,207679486~2890179,233701664~1501648,272948381~1516685,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.910 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 17 diff [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,201071067~625261,205246723~5322942,233701664~1501648,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.911 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,201071067~625261,205246723~5322942,233701664~1501648,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.917 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159680442~15240543,176051091~5895812,189908632~4104040,197786034~3910294,205246723~5322942,233701664~1501648,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.917 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 18 diff [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159680442~22266461,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.917 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159680442~22266461,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.923 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,77036241~6394957,84756138~4620842,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159680442~22266461,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.924 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 19 diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.924 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.930 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.930 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 13 diff [108462095~3696521,113389514~7808033,127746465~6902182,162125061~10387091,176420840~1173598,233701664~1501648,330570804~2468514,364820588~1893967] 2024-07-28T19:47:56.930 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [108462095~3696521,113389514~7808033,127746465~6902182,162125061~10387091,176420840~1173598,233701664~1501648,330570804~2468514,364820588~1893967] 2024-07-28T19:47:56.935 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [104857600~7301016,113389514~7808033,127746465~6902182,162125061~10387091,176420840~1173598,233701664~1501648,330570804~2468514,364820588~1893967] 2024-07-28T19:47:56.935 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 14 diff [38624001~3004469,98333376~1495812,108254853~12942694,127746465~6902182,139059271~4848819,162125061~12795924,176420840~1173598,201071067~625261,233701664~1501648,330570804~2468514,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.935 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [38624001~3004469,98333376~1495812,108254853~12942694,127746465~6902182,139059271~4848819,162125061~12795924,176420840~1173598,201071067~625261,233701664~1501648,330570804~2468514,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.941 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [38624001~3004469,98333376~1495812,104857600~16339947,127746465~6902182,139059271~4848819,162125061~12795924,176420840~1173598,197786034~3910294,233701664~1501648,330570804~2468514,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.941 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 15 diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,84756138~4620842,98333376~1495812,108254853~12942694,127746465~6902182,139059271~4848819,159680442~15240543,176420840~1173598,201071067~625261,233701664~1501648,308197699~851853,330570804~2468514,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.941 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,84756138~4620842,98333376~1495812,108254853~12942694,127746465~6902182,139059271~4848819,159680442~15240543,176420840~1173598,201071067~625261,233701664~1501648,308197699~851853,330570804~2468514,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.947 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,84756138~4620842,98333376~1495812,104857600~16339947,127746465~6902182,139059271~4848819,159680442~15240543,176420840~1173598,197786034~3910294,233701664~1501648,308197699~851853,330570804~2468514,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.947 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 16 diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,84756138~4620842,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~4848819,159680442~15240543,176163511~5783392,201071067~625261,233701664~1501648,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.947 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,84756138~4620842,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~4848819,159680442~15240543,176163511~5783392,201071067~625261,233701664~1501648,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.954 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,84756138~4620842,98333376~1495812,100663296~20534251,127746465~6902182,139059271~4848819,159680442~15240543,176163511~5783392,197786034~3910294,233701664~1501648,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.954 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 17 diff [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159680442~15240543,176163511~5783392,201071067~625261,205246723~4536920,233701664~1501648,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.954 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159680442~15240543,176163511~5783392,201071067~625261,205246723~4536920,233701664~1501648,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.960 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159680442~15240543,176163511~5783392,197786034~3910294,205246723~4536920,233701664~1501648,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.960 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 18 diff [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159680442~22266461,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.960 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159680442~22266461,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.967 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159680442~22266461,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.967 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 19 diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.968 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.973 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159391129~23307573,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.973 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 14 diff [38624001~3004469,98333376~1495812,108254853~6235284,139059271~4848819,171959564~2961421,201071067~625261,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.974 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [38624001~3004469,98333376~1495812,108254853~6235284,139059271~4848819,171959564~2961421,201071067~625261,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.979 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [38624001~3004469,98333376~1495812,108254853~6235284,139059271~4848819,171959564~2961421,197786034~3910294,363707150~7670438,399392058~7026463] 2024-07-28T19:47:56.979 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 15 diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,84756138~4620842,98333376~1495812,108254853~6235284,139059271~4848819,159680442~5214158,171959564~2961421,201071067~625261,308197699~851853,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.980 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,84756138~4620842,98333376~1495812,108254853~6235284,139059271~4848819,159680442~5214158,171959564~2961421,201071067~625261,308197699~851853,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.986 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,84756138~4620842,98333376~1495812,108254853~6235284,139059271~4848819,159680442~5214158,171959564~2961421,197786034~3910294,308197699~851853,360999874~2416421,363707150~7670438,393439548~12978973] 2024-07-28T19:47:56.986 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 16 diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,84756138~4620842,98333376~1495812,101072762~6550905,108254853~6235284,139059271~4848819,159680442~5214158,171959564~2961421,176163511~5783392,201071067~625261,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.986 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,84756138~4620842,98333376~1495812,101072762~6550905,108254853~6235284,139059271~4848819,159680442~5214158,171959564~2961421,176163511~5783392,201071067~625261,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.992 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,29732490~296310,38624001~3004469,53284707~5794772,67707114~4813819,84756138~4620842,98333376~1495812,100663296~6960371,108254853~6235284,139059271~4848819,159680442~5214158,171959564~2961421,176163511~5783392,197786034~3910294,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.992 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 17 diff [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159680442~5214158,171959564~2961421,176163511~5783392,201071067~625261,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.992 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159680442~5214158,171959564~2961421,176163511~5783392,201071067~625261,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:56 smithi100 bash[80199]: cluster 2024-07-28T19:47:55.729397+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-07-28T19:47:56.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:56 smithi100 bash[80199]: cluster 2024-07-28T19:47:56.043222+0000 mgr.y (mgr.25358) 460 : cluster [DBG] pgmap v952: 105 pgs: 105 active+clean; 750 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 229 KiB/s rd, 49 MiB/s wr, 359 op/s 2024-07-28T19:47:56.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:56 smithi100 bash[82644]: cluster 2024-07-28T19:47:55.729397+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-07-28T19:47:56.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:56 smithi100 bash[82644]: cluster 2024-07-28T19:47:56.043222+0000 mgr.y (mgr.25358) 460 : cluster [DBG] pgmap v952: 105 pgs: 105 active+clean; 750 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 229 KiB/s rd, 49 MiB/s wr, 359 op/s 2024-07-28T19:47:56.999 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,29732490~296310,38624001~3004469,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,100663296~6960371,108254853~6235284,139059271~9593941,159680442~5214158,171959564~2961421,176163511~5783392,197786034~3910294,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.999 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 18 diff [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159680442~5214158,171959564~9987339,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:56.999 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159680442~5214158,171959564~9987339,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.005 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,29732490~296310,38624001~3004469,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,98333376~1495812,100663296~6960371,108254853~6235284,139059271~9593941,159680442~5214158,171959564~9987339,197786034~3910294,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.005 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 19 diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.005 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.011 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~6960371,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,197786034~3910294,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.011 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 15 diff [992284~7616291,11026132~4097000,29732490~296310,53284707~5794772,84756138~4620842,159680442~5214158,308197699~851853,360999874~2416421,393439548~6441384] 2024-07-28T19:47:57.011 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,29732490~296310,53284707~5794772,84756138~4620842,159680442~5214158,308197699~851853,360999874~2416421,393439548~6441384] 2024-07-28T19:47:57.017 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,29732490~296310,53284707~5794772,84756138~4620842,159680442~5214158,308197699~851853,360999874~2416421,393439548~6441384] 2024-07-28T19:47:57.017 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 16 diff [992284~7616291,11026132~4097000,29732490~296310,53284707~5794772,67707114~4813819,84756138~4620842,101072762~6550905,159680442~5214158,176163511~5783392,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.017 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~7616291,11026132~4097000,29732490~296310,53284707~5794772,67707114~4813819,84756138~4620842,101072762~6550905,159680442~5214158,176163511~5783392,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.023 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~7616291,11026132~4097000,29732490~296310,53284707~5794772,67707114~4813819,84756138~4620842,100663296~6960371,159680442~5214158,176163511~5783392,272948381~1516685,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.023 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 17 diff [992284~14130848,29732490~296310,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,101072762~6550905,142610709~6042503,159680442~5214158,176163511~5783392,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.023 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,29732490~296310,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,101072762~6550905,142610709~6042503,159680442~5214158,176163511~5783392,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.029 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,29732490~296310,53164013~8315947,67707114~4813819,84756138~4620842,94703460~1486531,100663296~6960371,142610709~6042503,159680442~5214158,176163511~5783392,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.029 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 18 diff [992284~14130848,29732490~296310,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,101072762~6550905,141112044~7541168,159680442~5214158,174132279~7814624,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.030 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~14130848,29732490~296310,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,101072762~6550905,141112044~7541168,159680442~5214158,174132279~7814624,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.036 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~14130848,29732490~296310,42863566~319415,50422011~12314024,67707114~4813819,84756138~4620842,94703460~1486531,100663296~6960371,141112044~7541168,159680442~5214158,174132279~7814624,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.036 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 19 diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.036 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.042 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,373293056~3097881,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.042 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to 16 diff [67707114~4813819,101072762~6550905,176163511~5783392,272948381~1516685,288553325~3571025,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.042 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [67707114~4813819,101072762~6550905,176163511~5783392,272948381~1516685,288553325~3571025,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.048 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [67707114~4813819,100663296~6960371,176163511~5783392,272948381~1516685,288553325~3571025,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.048 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to 17 diff [4007687~7504201,11782144~1642460,53164013~536842,56120723~5359237,67707114~4813819,94703460~1486531,101072762~6550905,142610709~6042503,176163511~5783392,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.048 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,53164013~536842,56120723~5359237,67707114~4813819,94703460~1486531,101072762~6550905,142610709~6042503,176163511~5783392,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.054 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,53164013~536842,56120723~5359237,67707114~4813819,94703460~1486531,100663296~6960371,142610709~6042503,176163511~5783392,205246723~4536920,239968511~70796,272948381~1516685,275837321~6859657,288553325~3571025,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.054 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to 18 diff [4007687~7504201,11782144~1642460,42863566~319415,50422011~12314024,67707114~4813819,94703460~1486531,101072762~6550905,141112044~7541168,174132279~7814624,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.054 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,42863566~319415,50422011~12314024,67707114~4813819,94703460~1486531,101072762~6550905,141112044~7541168,174132279~7814624,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:56 smithi138 bash[67894]: cluster 2024-07-28T19:47:55.729397+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-07-28T19:47:57.056 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:56 smithi138 bash[67894]: cluster 2024-07-28T19:47:56.043222+0000 mgr.y (mgr.25358) 460 : cluster [DBG] pgmap v952: 105 pgs: 105 active+clean; 750 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 229 KiB/s rd, 49 MiB/s wr, 359 op/s 2024-07-28T19:47:57.060 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,42863566~319415,50422011~12314024,67707114~4813819,94703460~1486531,100663296~6960371,141112044~7541168,174132279~7814624,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.060 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to 19 diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,375306772~1084165,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.060 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,375306772~1084165,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.066 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~27861320,86448622~4240801,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,373293056~3097881,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.067 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to 17 diff [4007687~7504201,11782144~1642460,53164013~536842,56120723~5359237,94703460~1486531,142610709~6042503,205246723~4536920,239968511~70796,275837321~6859657] 2024-07-28T19:47:57.067 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,53164013~536842,56120723~5359237,94703460~1486531,142610709~6042503,205246723~4536920,239968511~70796,275837321~6859657] 2024-07-28T19:47:57.076 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,53164013~536842,56120723~5359237,94703460~1486531,142610709~6042503,205246723~4536920,239968511~70796,275837321~6859657] 2024-07-28T19:47:57.076 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to 18 diff [4007687~7504201,11782144~1642460,42863566~319415,50422011~12314024,94703460~1486531,141112044~7541168,174132279~2675167,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566] 2024-07-28T19:47:57.076 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,42863566~319415,50422011~12314024,94703460~1486531,141112044~7541168,174132279~2675167,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566] 2024-07-28T19:47:57.086 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,42863566~319415,50422011~12314024,94703460~1486531,141112044~7541168,174132279~2675167,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566] 2024-07-28T19:47:57.086 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to 19 diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,375306772~1084165] 2024-07-28T19:47:57.086 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,375306772~1084165] 2024-07-28T19:47:57.092 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~27861320,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,373293056~3097881] 2024-07-28T19:47:57.092 INFO:tasks.workunit.client.0.smithi100.stdout:from 17 to 18 diff [42863566~319415,50422011~6169396,60101578~2634457,141112044~3631280,174132279~2675167,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566] 2024-07-28T19:47:57.092 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [42863566~319415,50422011~6169396,60101578~2634457,141112044~3631280,174132279~2675167,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566] 2024-07-28T19:47:57.097 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [42863566~319415,50422011~6169396,60101578~2634457,141112044~3631280,174132279~2675167,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566] 2024-07-28T19:47:57.097 INFO:tasks.workunit.client.0.smithi100.stdout:from 17 to 19 diff [13965872~2199139,35955503~22983,42863566~319415,50422011~6169396,60101578~14620182,74931225~3352106,86448622~4240801,108627228~3624510,141112044~3631280,159391129~1478312,174132279~2675167,178193457~4505245,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566,375306772~1084165] 2024-07-28T19:47:57.097 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [13965872~2199139,35955503~22983,42863566~319415,50422011~6169396,60101578~14620182,74931225~3352106,86448622~4240801,108627228~3624510,141112044~3631280,159391129~1478312,174132279~2675167,178193457~4505245,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566,375306772~1084165] 2024-07-28T19:47:57.109 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [13965872~2199139,35955503~22983,42863566~319415,50422011~6169396,60101578~18181753,86448622~4240801,108627228~3624510,141112044~3631280,159391129~1478312,174132279~2675167,178193457~4505245,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566,373293056~3097881] 2024-07-28T19:47:57.109 INFO:tasks.workunit.client.0.smithi100.stdout:from 18 to 19 diff [13965872~2199139,35955503~22983,61441917~13279843,74931225~3352106,86448622~4240801,108627228~3624510,159391129~1478312,178193457~4505245,375306772~1084165] 2024-07-28T19:47:57.109 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [13965872~2199139,35955503~22983,61441917~13279843,74931225~3352106,86448622~4240801,108627228~3624510,159391129~1478312,178193457~4505245,375306772~1084165] 2024-07-28T19:47:57.115 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [13965872~2199139,35955503~22983,61441917~16841414,86448622~4240801,108627228~3624510,159391129~1478312,178193457~4505245,373293056~3097881] 2024-07-28T19:47:57.765 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:57.765 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:57.804 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:47:57.804 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:57.804 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:57.818 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:47:57.818 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:57.818 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:57.830 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:47:57.830 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:57.831 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:57.842 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:47:57.843 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:47:57.843 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:47:57.853 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:47:57.853 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:47:57.853 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:47:57.864 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:47:57.864 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:57.864 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:57.873 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:57.873 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:57.873 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:57.881 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:57.882 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:57.882 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:57.890 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:57.890 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:47:57.890 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:47:57.897 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:47:57.897 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.897 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.906 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.906 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.906 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.910 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.910 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.911 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.917 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159391129~23307573,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.917 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.917 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.923 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~6960371,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,197786034~3910294,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:57.923 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.923 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.930 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,373293056~3097881,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:57.930 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to HEAD diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,375306772~1084165,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.930 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,375306772~1084165,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.936 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~27861320,86448622~4240801,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,373293056~3097881,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:57.937 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to HEAD diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,375306772~1084165] 2024-07-28T19:47:57.937 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,375306772~1084165] 2024-07-28T19:47:57.943 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~27861320,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,373293056~3097881] 2024-07-28T19:47:57.943 INFO:tasks.workunit.client.0.smithi100.stdout:from 17 to HEAD diff [13965872~2199139,35955503~22983,42863566~319415,50422011~6169396,60101578~14620182,74931225~3352106,86448622~4240801,108627228~3624510,141112044~3631280,159391129~1478312,174132279~2675167,178193457~4505245,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566,375306772~1084165] 2024-07-28T19:47:57.943 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [13965872~2199139,35955503~22983,42863566~319415,50422011~6169396,60101578~14620182,74931225~3352106,86448622~4240801,108627228~3624510,141112044~3631280,159391129~1478312,174132279~2675167,178193457~4505245,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566,375306772~1084165] 2024-07-28T19:47:57.949 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [13965872~2199139,35955503~22983,42863566~319415,50422011~6169396,60101578~18181753,86448622~4240801,108627228~3624510,141112044~3631280,159391129~1478312,174132279~2675167,178193457~4505245,208065608~5472099,221190067~5571570,280271725~3919291,293406177~7064566,373293056~3097881] 2024-07-28T19:47:58.783 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:58.783 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:58.815 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:47:58.815 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:58.816 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:58.829 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:47:58.829 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:58.829 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:58.841 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:47:58.841 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:58.841 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:58.852 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:47:58.852 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:47:58.852 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:47:58.862 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:47:58.862 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:47:58.862 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:47:58.871 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:47:58.871 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:58.871 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:58.880 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:58.881 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:58.881 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:58.889 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:58.889 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:58.889 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:58.896 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:58.897 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:47:58.897 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:47:58.903 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:47:58.904 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.904 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.910 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.911 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.911 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.917 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.917 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.917 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.926 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159391129~23307573,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.926 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.926 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.930 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~6960371,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,197786034~3910294,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:58.930 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:58.930 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:58.936 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,373293056~3097881,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:58.936 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to HEAD diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,375306772~1084165,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:58.936 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,375306772~1084165,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:58.942 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~27861320,86448622~4240801,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,373293056~3097881,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:58.942 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to HEAD diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,375306772~1084165] 2024-07-28T19:47:58.942 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,375306772~1084165] 2024-07-28T19:47:58.948 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~27861320,86448622~4240801,94703460~1486531,108627228~3624510,141112044~7541168,159391129~1478312,174132279~2675167,178193457~4505245,205246723~8290984,221190067~5571570,239968511~70796,275837321~8353695,293406177~7064566,373293056~3097881] 2024-07-28T19:47:59.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:58 smithi138 bash[67894]: cluster 2024-07-28T19:47:57.758165+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-07-28T19:47:59.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:58 smithi138 bash[67894]: cluster 2024-07-28T19:47:58.043923+0000 mgr.y (mgr.25358) 461 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 780 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 225 KiB/s rd, 23 MiB/s wr, 1.58k op/s 2024-07-28T19:47:59.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:58 smithi100 bash[80199]: cluster 2024-07-28T19:47:57.758165+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-07-28T19:47:59.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:58 smithi100 bash[80199]: cluster 2024-07-28T19:47:58.043923+0000 mgr.y (mgr.25358) 461 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 780 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 225 KiB/s rd, 23 MiB/s wr, 1.58k op/s 2024-07-28T19:47:59.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:58 smithi100 bash[82644]: cluster 2024-07-28T19:47:57.758165+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-07-28T19:47:59.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:58 smithi100 bash[82644]: cluster 2024-07-28T19:47:58.043923+0000 mgr.y (mgr.25358) 461 : cluster [DBG] pgmap v954: 105 pgs: 105 active+clean; 780 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 225 KiB/s rd, 23 MiB/s wr, 1.58k op/s 2024-07-28T19:47:59.804 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:59.804 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:59.831 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:47:59.831 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:59.831 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:47:59.843 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:47:59.843 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:59.843 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:59.855 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:47:59.855 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:59.855 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:47:59.866 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:47:59.866 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:47:59.866 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:47:59.876 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:47:59.876 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:47:59.876 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:47:59.885 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:47:59.885 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:59.886 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:59.893 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:59.893 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:59.893 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:59.901 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:59.901 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:59.901 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:47:59.908 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:47:59.909 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:47:59.909 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:47:59.916 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:47:59.916 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.916 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.923 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.923 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.924 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.930 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.931 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.931 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.937 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159391129~23307573,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.938 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.938 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.943 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~6960371,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,197786034~3910294,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:47:59.944 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:59.944 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:59.950 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,373293056~3097881,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:47:59.950 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to HEAD diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,375306772~1084165,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:59.950 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,86448622~4240801,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,375306772~1084165,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:47:59.956 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4007687~7504201,11782144~1642460,13965872~2199139,35955503~22983,42863566~319415,50422011~27861320,86448622~4240801,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~1478312,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,326277743~7799913,348871796~1787352,373293056~3097881,383067522~4363904,396069127~4050808,400701040~7300403] 2024-07-28T19:48:00.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:59 smithi138 bash[67894]: cluster 2024-07-28T19:47:58.768206+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-07-28T19:48:00.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:47:59 smithi138 bash[67894]: audit 2024-07-28T19:47:58.769239+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:00.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:59 smithi100 bash[80199]: cluster 2024-07-28T19:47:58.768206+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-07-28T19:48:00.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:47:59 smithi100 bash[80199]: audit 2024-07-28T19:47:58.769239+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:00.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:59 smithi100 bash[82644]: cluster 2024-07-28T19:47:58.768206+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-07-28T19:48:00.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:47:59 smithi100 bash[82644]: audit 2024-07-28T19:47:58.769239+0000 mon.a (mon.0) 1431 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:00.823 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:00.823 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:00.851 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:00.851 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:00.851 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:00.861 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:00.861 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:00.862 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:00.871 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:00.871 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:00.871 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:00.880 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:00.880 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:00.880 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:00.888 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:00.888 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:00.888 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:00.896 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:00.896 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:00.896 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:00.903 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:00.903 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:00.904 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:00.910 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:00.910 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:00.910 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:00.917 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:00.917 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:00.917 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:00.923 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:48:00.923 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.923 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.929 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.930 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.930 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.936 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.936 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.936 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.942 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159391129~23307573,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.942 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.942 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.948 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~6960371,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,197786034~3910294,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:00.948 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:48:00.948 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,101072762~6550905,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,375306772~1084165,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:48:00.954 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,100663296~6960371,108627228~3624510,141112044~7541168,159391129~5503471,174132279~8566423,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,373293056~3097881,383067522~4363904,393439548~6680387,400701040~7300403] 2024-07-28T19:48:01.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:00 smithi138 bash[67894]: cluster 2024-07-28T19:47:59.767918+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-07-28T19:48:01.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:00 smithi138 bash[67894]: cluster 2024-07-28T19:48:00.044602+0000 mgr.y (mgr.25358) 462 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 812 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 168 KiB/s rd, 15 MiB/s wr, 4.44k op/s 2024-07-28T19:48:01.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:00 smithi100 bash[80199]: cluster 2024-07-28T19:47:59.767918+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-07-28T19:48:01.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:00 smithi100 bash[80199]: cluster 2024-07-28T19:48:00.044602+0000 mgr.y (mgr.25358) 462 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 812 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 168 KiB/s rd, 15 MiB/s wr, 4.44k op/s 2024-07-28T19:48:01.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:00 smithi100 bash[82644]: cluster 2024-07-28T19:47:59.767918+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-07-28T19:48:01.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:00 smithi100 bash[82644]: cluster 2024-07-28T19:48:00.044602+0000 mgr.y (mgr.25358) 462 : cluster [DBG] pgmap v957: 105 pgs: 105 active+clean; 812 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 168 KiB/s rd, 15 MiB/s wr, 4.44k op/s 2024-07-28T19:48:01.813 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:01.813 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:01.859 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:01.859 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:01.859 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:01.869 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:01.869 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:01.869 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:01.878 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:01.878 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:01.878 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:01.886 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:01.886 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:01.886 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:01.894 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:01.894 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:01.895 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:01.902 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:01.902 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:01.902 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:01.909 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:01.909 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:01.909 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:01.916 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:01.917 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:01.917 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:01.923 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:01.923 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:01.923 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:01.930 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:48:01.930 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.930 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.936 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.936 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.936 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.943 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.943 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.943 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.949 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159391129~23307573,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.949 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.949 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,201071067~625261,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:01.955 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~6960371,108254853~6235284,139059271~9593941,159391129~5503471,171959564~10739138,197786034~3910294,205246723~8290984,221190067~5571570,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:01 smithi138 bash[67894]: cluster 2024-07-28T19:48:00.792795+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-07-28T19:48:02.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:01 smithi100 bash[80199]: cluster 2024-07-28T19:48:00.792795+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-07-28T19:48:02.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:01 smithi100 bash[82644]: cluster 2024-07-28T19:48:00.792795+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-07-28T19:48:02.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:48:02 smithi138 bash[60615]: ts=2024-07-28T19:48:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:48:02.852 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:02.853 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:02.887 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:02.887 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:02.887 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:02.897 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:02.898 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:02.898 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:02.907 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:02.907 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:02.907 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:02.916 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:02.916 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:02.916 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:02.924 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:02.924 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:02.925 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:02.932 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:02.933 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:02.933 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:02.940 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:02.940 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:02.940 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:02.947 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:02.947 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:02.947 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:02.954 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:02.954 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:02.955 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:02.961 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:48:02.961 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.961 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.968 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.968 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.968 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.974 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.974 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.974 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~3352106,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,159391129~23307573,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:02.980 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~27861320,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,159391129~23307573,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,308197699~851853,326277743~7799913,348871796~1787352,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:03.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:48:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:48:02] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2024-07-28T19:48:03.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:02 smithi100 bash[80199]: cluster 2024-07-28T19:48:01.793244+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-07-28T19:48:03.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:02 smithi100 bash[80199]: cluster 2024-07-28T19:48:02.045247+0000 mgr.y (mgr.25358) 463 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 811 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 56 KiB/s rd, 8.6 MiB/s wr, 3.94k op/s 2024-07-28T19:48:03.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:02 smithi100 bash[82644]: cluster 2024-07-28T19:48:01.793244+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-07-28T19:48:03.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:02 smithi100 bash[82644]: cluster 2024-07-28T19:48:02.045247+0000 mgr.y (mgr.25358) 463 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 811 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 56 KiB/s rd, 8.6 MiB/s wr, 3.94k op/s 2024-07-28T19:48:03.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:02 smithi138 bash[67894]: cluster 2024-07-28T19:48:01.793244+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-07-28T19:48:03.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:02 smithi138 bash[67894]: cluster 2024-07-28T19:48:02.045247+0000 mgr.y (mgr.25358) 463 : cluster [DBG] pgmap v960: 105 pgs: 105 active+clean; 811 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 56 KiB/s rd, 8.6 MiB/s wr, 3.94k op/s 2024-07-28T19:48:03.851 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:03.852 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:03.892 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:03.892 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:03.892 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:03.901 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:03.901 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:03.901 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:03.910 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:03.910 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:03.910 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:03.917 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:03.917 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:03.918 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:03.924 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:03.924 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:03.924 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:03.931 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:03.931 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:03.931 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:03.938 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:03.938 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:03.938 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:03.944 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:03.944 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:03.945 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:03.951 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:03.951 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:03.951 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:03.957 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:48:03.957 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:03.957 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:03.963 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:03.963 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:03.963 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~12942694,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:03.969 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,29732490~296310,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~20534251,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~8290984,221190067~5571570,233701664~1501648,239968511~70796,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:04.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:03 smithi100 bash[80199]: cluster 2024-07-28T19:48:02.823721+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-07-28T19:48:04.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:03 smithi100 bash[82644]: cluster 2024-07-28T19:48:02.823721+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-07-28T19:48:04.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:03 smithi138 bash[67894]: cluster 2024-07-28T19:48:02.823721+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-07-28T19:48:04.876 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:04.876 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:04.911 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:04.911 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:04.911 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:04.920 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:04.920 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:04.920 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:04.929 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:04.929 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:04.929 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:04.937 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:04.937 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:04.938 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:04.945 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:04.945 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:04.945 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:04.952 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:04.953 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:04.953 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:04.959 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:04.959 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:04.959 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:04.966 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:04.966 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:04.966 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:04.973 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:04.973 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:04.973 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:04.980 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:48:04.980 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:04.980 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~24299749,74931225~8499973,84756138~5933285,94703460~1486531,98333376~1495812,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,201071067~625261,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,393439548~14561895] 2024-07-28T19:48:04.986 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,50422011~33009187,84756138~5933285,94703460~1486531,98333376~1495812,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~23307573,189908632~4104040,197786034~3910294,205246723~9994291,221190067~5571570,233701664~1501648,235936831~3095605,239968511~70796,256606700~476669,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,393439548~14561895] 2024-07-28T19:48:05.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:04 smithi100 bash[80199]: cluster 2024-07-28T19:48:03.822807+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-07-28T19:48:05.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:04 smithi100 bash[80199]: cluster 2024-07-28T19:48:04.046047+0000 mgr.y (mgr.25358) 464 : cluster [DBG] pgmap v963: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 777 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 148 KiB/s rd, 6.2 KiB/s wr, 2.10k op/s 2024-07-28T19:48:05.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:04 smithi100 bash[82644]: cluster 2024-07-28T19:48:03.822807+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-07-28T19:48:05.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:04 smithi100 bash[82644]: cluster 2024-07-28T19:48:04.046047+0000 mgr.y (mgr.25358) 464 : cluster [DBG] pgmap v963: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 777 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 148 KiB/s rd, 6.2 KiB/s wr, 2.10k op/s 2024-07-28T19:48:05.307 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:05.307 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:05.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:04 smithi138 bash[67894]: cluster 2024-07-28T19:48:03.822807+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-07-28T19:48:05.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:04 smithi138 bash[67894]: cluster 2024-07-28T19:48:04.046047+0000 mgr.y (mgr.25358) 464 : cluster [DBG] pgmap v963: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 777 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 148 KiB/s rd, 6.2 KiB/s wr, 2.10k op/s 2024-07-28T19:48:05.340 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:05.340 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:05.340 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:05.352 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:05.352 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:05.352 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:05.359 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:05.359 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:05.359 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:05.367 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:05.367 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:05.367 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:05.374 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:05.374 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:05.374 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:05.381 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:05.381 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:05.381 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:05.388 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:05.388 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:05.388 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:05.394 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:05.395 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:05.395 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:05.401 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:05.401 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:05.401 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,375306772~1084165,383067522~4363904,391211106~16790337] 2024-07-28T19:48:05.407 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139059271~9593941,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,346870513~3788635,355162073~1084006,360999874~2416421,363707150~7670438,373293056~3097881,383067522~4363904,391211106~16790337] 2024-07-28T19:48:06.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:05 smithi100 bash[82644]: cluster 2024-07-28T19:48:04.843792+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-07-28T19:48:06.249 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:05 smithi100 bash[82644]: cluster 2024-07-28T19:48:05.266873+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-07-28T19:48:06.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:05 smithi100 bash[80199]: cluster 2024-07-28T19:48:04.843792+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-07-28T19:48:06.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:05 smithi100 bash[80199]: cluster 2024-07-28T19:48:05.266873+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-07-28T19:48:06.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:05 smithi138 bash[67894]: cluster 2024-07-28T19:48:04.843792+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-07-28T19:48:06.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:05 smithi138 bash[67894]: cluster 2024-07-28T19:48:05.266873+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-07-28T19:48:06.315 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:06.315 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:06.358 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:06.358 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:06.358 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:06.367 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:06.367 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:06.367 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:06.375 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:06.375 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:06.375 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:06.383 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:06.383 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:06.383 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:06.391 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:06.391 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:06.391 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:06.398 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:06.398 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:06.398 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:06.405 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:06.405 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:06.405 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:06.412 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:06.412 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:06.412 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~28927102,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:06.419 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20780229~2485981,26493107~2633969,29732490~296310,32790789~1112582,35955503~22983,38624001~3004469,42863566~319415,45794658~37636540,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,253887767~3238189,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:07.311 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:07.312 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:07.360 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:07.360 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:07.360 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:07.370 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:07.370 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:07.370 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:07.379 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:07.379 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:07.379 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:07.386 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:07.386 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:07.386 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:07.393 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:07.393 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:07.393 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:07.400 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:07.400 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:07.400 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:07.407 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:07.407 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:07.407 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246992343~10133613,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:07.413 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,127746465~6902182,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~5571570,231922261~7110175,239968511~70796,246603086~10522870,272948381~1516685,275837321~8353695,288553325~3571025,293406177~7064566,304100381~6208439,311478804~409659,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:07.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:07 smithi138 bash[67894]: cluster 2024-07-28T19:48:06.046760+0000 mgr.y (mgr.25358) 465 : cluster [DBG] pgmap v966: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 762 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 143 KiB/s rd, 6.2 KiB/s wr, 2.00k op/s 2024-07-28T19:48:07.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:07 smithi138 bash[67894]: cluster 2024-07-28T19:48:06.283213+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-07-28T19:48:07.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:07 smithi100 bash[80199]: cluster 2024-07-28T19:48:06.046760+0000 mgr.y (mgr.25358) 465 : cluster [DBG] pgmap v966: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 762 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 143 KiB/s rd, 6.2 KiB/s wr, 2.00k op/s 2024-07-28T19:48:07.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:07 smithi100 bash[80199]: cluster 2024-07-28T19:48:06.283213+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-07-28T19:48:07.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:07 smithi100 bash[82644]: cluster 2024-07-28T19:48:06.046760+0000 mgr.y (mgr.25358) 465 : cluster [DBG] pgmap v966: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 762 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 143 KiB/s rd, 6.2 KiB/s wr, 2.00k op/s 2024-07-28T19:48:07.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:07 smithi100 bash[82644]: cluster 2024-07-28T19:48:06.283213+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-07-28T19:48:08.333 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:08.334 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:08.386 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:08.386 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:08.386 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:08.395 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:08.395 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:08.395 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:08.403 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:08.403 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:08.403 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:08.411 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:08.411 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:08.411 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:08.418 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:08.418 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:08.418 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:08.425 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:08.425 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:08.425 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246992343~10133613,260707075~4572808,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921] 2024-07-28T19:48:08.431 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,20182543~3982629,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,221190067~6368989,231922261~7110175,239968511~70796,246603086~10522870,260046848~5233035,272948381~1516685,274828887~893301,275837321~8353695,288553325~3571025,293406177~7064566,304100381~12314078,326277743~7799913,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921] 2024-07-28T19:48:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:08 smithi138 bash[67894]: cluster 2024-07-28T19:48:07.284628+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-07-28T19:48:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:08 smithi138 bash[67894]: audit 2024-07-28T19:48:07.643649+0000 mon.a (mon.0) 1441 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:08 smithi100 bash[80199]: cluster 2024-07-28T19:48:07.284628+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-07-28T19:48:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:08 smithi100 bash[80199]: audit 2024-07-28T19:48:07.643649+0000 mon.a (mon.0) 1441 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:08.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:08 smithi100 bash[82644]: cluster 2024-07-28T19:48:07.284628+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-07-28T19:48:08.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:08 smithi100 bash[82644]: audit 2024-07-28T19:48:07.643649+0000 mon.a (mon.0) 1441 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:09.347 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:09.347 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:09.390 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:09.390 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:09.390 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:09.401 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:09.401 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:09.401 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:09.409 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:09.409 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:09.410 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:09.417 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:09.418 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:09.418 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:09.425 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:09.425 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:09.425 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [992284~15172727,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~24933921,410737048~1385636] 2024-07-28T19:48:09.432 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [992284~15172727,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,153891366~718136,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~24933921,410737048~1385636] 2024-07-28T19:48:09.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:09 smithi100 bash[80199]: cluster 2024-07-28T19:48:08.047591+0000 mgr.y (mgr.25358) 466 : cluster [DBG] pgmap v969: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 738 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 123 KiB/s rd, 5.2 KiB/s wr, 1.15k op/s 2024-07-28T19:48:09.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:09 smithi100 bash[80199]: cluster 2024-07-28T19:48:08.311885+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-07-28T19:48:09.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:09 smithi100 bash[82644]: cluster 2024-07-28T19:48:08.047591+0000 mgr.y (mgr.25358) 466 : cluster [DBG] pgmap v969: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 738 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 123 KiB/s rd, 5.2 KiB/s wr, 1.15k op/s 2024-07-28T19:48:09.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:09 smithi100 bash[82644]: cluster 2024-07-28T19:48:08.311885+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-07-28T19:48:09.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:09 smithi138 bash[67894]: cluster 2024-07-28T19:48:08.047591+0000 mgr.y (mgr.25358) 466 : cluster [DBG] pgmap v969: 105 pgs: 2 active+clean+snaptrim, 103 active+clean; 738 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 123 KiB/s rd, 5.2 KiB/s wr, 1.15k op/s 2024-07-28T19:48:09.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:09 smithi138 bash[67894]: cluster 2024-07-28T19:48:08.311885+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-07-28T19:48:10.297 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:10.298 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:10.343 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:10.343 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:10.343 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:10.352 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:10.352 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:10.352 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:10.359 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:10.359 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:10.360 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:10.367 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:10.367 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:10.367 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,201071067~625261,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,383067522~26060898,410737048~1385636] 2024-07-28T19:48:10.373 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,189908632~4104040,197786034~3910294,205246723~9994291,215999575~1776727,217934213~1252500,220349894~7209162,231922261~7110175,239968511~70796,246603086~10522870,260046848~8477691,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,383067522~26060898,410737048~1385636] 2024-07-28T19:48:10.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:10 smithi100 bash[80199]: cluster 2024-07-28T19:48:09.314667+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-07-28T19:48:10.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:10 smithi100 bash[80199]: cluster 2024-07-28T19:48:10.270505+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-07-28T19:48:10.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:10 smithi100 bash[82644]: cluster 2024-07-28T19:48:09.314667+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-07-28T19:48:10.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:10 smithi100 bash[82644]: cluster 2024-07-28T19:48:10.270505+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-07-28T19:48:10.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:10 smithi138 bash[67894]: cluster 2024-07-28T19:48:09.314667+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-07-28T19:48:10.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:10 smithi138 bash[67894]: cluster 2024-07-28T19:48:10.270505+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-07-28T19:48:11.383 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:11.383 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:11.413 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:11.413 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:11.414 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:11.422 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:11.422 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:11.422 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:11.431 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:11.431 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:11.431 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~8353695,287356099~4768251,293406177~7064566,304100381~12314078,320568422~13509234,337540878~4423331,346870513~3788635,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:11.438 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~12920855,287356099~4768251,293406177~7064566,304100381~12314078,319519117~14558539,335544320~6419889,346870513~3788635,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:11.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:11 smithi100 bash[80199]: cluster 2024-07-28T19:48:10.048397+0000 mgr.y (mgr.25358) 467 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 663 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 6.7 KiB/s wr, 1.37k op/s 2024-07-28T19:48:11.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:11 smithi100 bash[82644]: cluster 2024-07-28T19:48:10.048397+0000 mgr.y (mgr.25358) 467 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 663 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 6.7 KiB/s wr, 1.37k op/s 2024-07-28T19:48:11.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:11 smithi138 bash[67894]: cluster 2024-07-28T19:48:10.048397+0000 mgr.y (mgr.25358) 467 : cluster [DBG] pgmap v972: 105 pgs: 105 active+clean; 663 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 127 KiB/s rd, 6.7 KiB/s wr, 1.37k op/s 2024-07-28T19:48:12.388 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:12.388 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:12.430 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:12.430 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:12.430 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:12.439 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:12.439 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:12.439 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,231922261~7110175,239968511~70796,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,355162073~1084006,357603777~13773811,375306772~1084165,381707621~27420799,410737048~1385636] 2024-07-28T19:48:12.446 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,26493107~9197440,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,197786034~29773022,230686720~8345716,239968511~70796,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~7299693,355162073~1084006,357603777~13773811,373293056~3097881,381681664~27446756,410737048~1385636] 2024-07-28T19:48:12.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:12 smithi100 bash[80199]: cluster 2024-07-28T19:48:11.350956+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-07-28T19:48:12.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:12 smithi100 bash[82644]: cluster 2024-07-28T19:48:11.350956+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-07-28T19:48:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:12 smithi138 bash[67894]: cluster 2024-07-28T19:48:11.350956+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-07-28T19:48:12.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:48:12 smithi138 bash[60615]: ts=2024-07-28T19:48:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:48:13.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:48:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:48:12] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2024-07-28T19:48:13.394 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:13.394 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:13.431 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:13.431 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:13.431 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~9197440,35955503~22983,38624001~36097759,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~9603824,149537883~2616756,152843223~4376530,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~9411927,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~13773811,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:13.440 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~10524723,35955503~22983,38624001~44807197,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,139049388~9603824,149537883~2616756,152843223~4376530,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~10647468,246603086~10522870,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~13773811,373293056~3097881,381681664~31785520] 2024-07-28T19:48:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:13 smithi100 bash[80199]: cluster 2024-07-28T19:48:12.049210+0000 mgr.y (mgr.25358) 468 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 655 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 1.5 KiB/s wr, 915 op/s 2024-07-28T19:48:13.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:13 smithi100 bash[80199]: cluster 2024-07-28T19:48:12.366338+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-07-28T19:48:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:13 smithi100 bash[82644]: cluster 2024-07-28T19:48:12.049210+0000 mgr.y (mgr.25358) 468 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 655 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 1.5 KiB/s wr, 915 op/s 2024-07-28T19:48:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:13 smithi100 bash[82644]: cluster 2024-07-28T19:48:12.366338+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-07-28T19:48:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:13 smithi138 bash[67894]: cluster 2024-07-28T19:48:12.049210+0000 mgr.y (mgr.25358) 468 : cluster [DBG] pgmap v975: 105 pgs: 105 active+clean; 655 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 1.5 KiB/s wr, 915 op/s 2024-07-28T19:48:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:13 smithi138 bash[67894]: cluster 2024-07-28T19:48:12.366338+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-07-28T19:48:14.408 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:14.408 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [112226~16052785,16897474~3285069,20365656~3799516,26493107~48228653,74931225~8499973,84756138~5933285,92287280~7627963,101072762~6550905,108254853~16591872,126138082~8510565,139049388~18170365,159391129~26141838,188958486~5054186,201071067~625261,201778320~13462694,215999575~11559481,229242050~1098587,231922261~14680825,246992343~10133613,260707075~7817464,271270161~1527390,272948381~1516685,274828887~893301,275837321~11052932,287356099~29058360,320568422~13509234,337540878~4423331,343359455~7299693,351158777~5921015,357603777~14068665,375306772~1084165,381707621~27420799,409516070~3951114] 2024-07-28T19:48:14.447 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16165011,16777216~7387956,25165824~58265374,83886080~6803343,92274688~7640555,100663296~24183429,125829120~8819527,138412032~18807721,159383552~26149415,188958486~5054186,197786034~29773022,228354301~1986336,230686720~26439236,260046848~8477691,271270161~45144298,319519117~14558539,335544320~6419889,343359455~13720337,357603777~14068665,373293056~3097881,381681664~31785520] 2024-07-28T19:48:14.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:14 smithi100 bash[80199]: cluster 2024-07-28T19:48:13.361795+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-07-28T19:48:14.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:14 smithi100 bash[82644]: cluster 2024-07-28T19:48:13.361795+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-07-28T19:48:14.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:14 smithi138 bash[67894]: cluster 2024-07-28T19:48:13.361795+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-07-28T19:48:15.328 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (42923 ms) 2024-07-28T19:48:15.329 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateRegression6926 2024-07-28T19:48:15.329 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:15.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:15 smithi100 bash[80199]: cluster 2024-07-28T19:48:14.050309+0000 mgr.y (mgr.25358) 469 : cluster [DBG] pgmap v978: 105 pgs: 2 active+clean+snaptrim, 5 active+clean+snaptrim_wait, 98 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 5.2 KiB/s wr, 574 op/s 2024-07-28T19:48:15.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:15 smithi100 bash[80199]: cluster 2024-07-28T19:48:14.384616+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-07-28T19:48:15.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:15 smithi100 bash[80199]: cluster 2024-07-28T19:48:15.271290+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-07-28T19:48:15.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:15 smithi100 bash[82644]: cluster 2024-07-28T19:48:14.050309+0000 mgr.y (mgr.25358) 469 : cluster [DBG] pgmap v978: 105 pgs: 2 active+clean+snaptrim, 5 active+clean+snaptrim_wait, 98 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 5.2 KiB/s wr, 574 op/s 2024-07-28T19:48:15.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:15 smithi100 bash[82644]: cluster 2024-07-28T19:48:14.384616+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-07-28T19:48:15.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:15 smithi100 bash[82644]: cluster 2024-07-28T19:48:15.271290+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-07-28T19:48:15.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:15 smithi138 bash[67894]: cluster 2024-07-28T19:48:14.050309+0000 mgr.y (mgr.25358) 469 : cluster [DBG] pgmap v978: 105 pgs: 2 active+clean+snaptrim, 5 active+clean+snaptrim_wait, 98 active+clean; 559 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 5.2 KiB/s wr, 574 op/s 2024-07-28T19:48:15.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:15 smithi138 bash[67894]: cluster 2024-07-28T19:48:14.384616+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-07-28T19:48:15.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:15 smithi138 bash[67894]: cluster 2024-07-28T19:48:15.271290+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-07-28T19:48:16.310 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~256 2024-07-28T19:48:16.321 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateRegression6926 (992 ms) 2024-07-28T19:48:16.321 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-07-28T19:48:16.321 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:17.545 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:48:17.545 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 4194304~4194304 2024-07-28T19:48:17.545 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 8388608~4194304 2024-07-28T19:48:17.545 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 12582912~4194304 2024-07-28T19:48:17.545 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 16777216~4194304 2024-07-28T19:48:17.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:17 smithi138 bash[67894]: cluster 2024-07-28T19:48:16.051108+0000 mgr.y (mgr.25358) 470 : cluster [DBG] pgmap v981: 105 pgs: 2 active+clean+snaptrim, 5 active+clean+snaptrim_wait, 98 active+clean; 512 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 5.2 KiB/s wr, 600 op/s 2024-07-28T19:48:17.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:17 smithi138 bash[67894]: cluster 2024-07-28T19:48:16.273833+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-07-28T19:48:17.618 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:48:17.618 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 4194304~4194304 2024-07-28T19:48:17.618 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 8388608~2097152 2024-07-28T19:48:17.618 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 16777216~4194304 2024-07-28T19:48:17.645 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1324 ms) 2024-07-28T19:48:17.645 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-07-28T19:48:17.645 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:17.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:17 smithi100 bash[80199]: cluster 2024-07-28T19:48:16.051108+0000 mgr.y (mgr.25358) 470 : cluster [DBG] pgmap v981: 105 pgs: 2 active+clean+snaptrim, 5 active+clean+snaptrim_wait, 98 active+clean; 512 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 5.2 KiB/s wr, 600 op/s 2024-07-28T19:48:17.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:17 smithi100 bash[80199]: cluster 2024-07-28T19:48:16.273833+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-07-28T19:48:17.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:17 smithi100 bash[82644]: cluster 2024-07-28T19:48:16.051108+0000 mgr.y (mgr.25358) 470 : cluster [DBG] pgmap v981: 105 pgs: 2 active+clean+snaptrim, 5 active+clean+snaptrim_wait, 98 active+clean; 512 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 5.2 KiB/s wr, 600 op/s 2024-07-28T19:48:17.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:17 smithi100 bash[82644]: cluster 2024-07-28T19:48:16.273833+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-07-28T19:48:18.653 INFO:tasks.workunit.client.0.smithi100.stdout: wrote [786114~63174,2879472~45912,3751736~30324,7388661~66189,11338234~12466,11989731~27305,16780620~57322,18337444~7988,18971061~20251,20042293~69531] to clone 2024-07-28T19:48:18.654 INFO:tasks.workunit.client.0.smithi100.stdout: diff was [0~3782060,4194304~3260546,8388608~3628428,16777216~3334608] 2024-07-28T19:48:18.665 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1020 ms) 2024-07-28T19:48:18.665 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-07-28T19:48:18.665 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:18.716 INFO:tasks.workunit.client.0.smithi100.stdout: wrote [1294519~10834,1560074~43773,3371116~83153,4966288~58937,11066774~87411,11867942~79571,12961881~26698,19156541~34843,20452799~75665,20714440~64843] 2024-07-28T19:48:18.745 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (80 ms) 2024-07-28T19:48:18.745 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-07-28T19:48:18.745 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:18.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:18 smithi100 bash[80199]: cluster 2024-07-28T19:48:17.389909+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-07-28T19:48:18.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:18 smithi100 bash[82644]: cluster 2024-07-28T19:48:17.389909+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-07-28T19:48:18.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:18 smithi138 bash[67894]: cluster 2024-07-28T19:48:17.389909+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-07-28T19:48:19.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:19 smithi100 bash[80199]: cluster 2024-07-28T19:48:18.052091+0000 mgr.y (mgr.25358) 471 : cluster [DBG] pgmap v984: 105 pgs: 105 active+clean; 478 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 3.2 KiB/s wr, 226 op/s 2024-07-28T19:48:19.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:19 smithi100 bash[80199]: cluster 2024-07-28T19:48:18.399926+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-07-28T19:48:19.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:19 smithi100 bash[82644]: cluster 2024-07-28T19:48:18.052091+0000 mgr.y (mgr.25358) 471 : cluster [DBG] pgmap v984: 105 pgs: 105 active+clean; 478 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 3.2 KiB/s wr, 226 op/s 2024-07-28T19:48:19.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:19 smithi100 bash[82644]: cluster 2024-07-28T19:48:18.399926+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-07-28T19:48:19.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:19 smithi138 bash[67894]: cluster 2024-07-28T19:48:18.052091+0000 mgr.y (mgr.25358) 471 : cluster [DBG] pgmap v984: 105 pgs: 105 active+clean; 478 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 3.2 KiB/s wr, 226 op/s 2024-07-28T19:48:19.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:19 smithi138 bash[67894]: cluster 2024-07-28T19:48:18.399926+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-07-28T19:48:20.510 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1765 ms) 2024-07-28T19:48:20.510 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-07-28T19:48:20.511 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:20.607 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 5000005~1234 2024-07-28T19:48:20.616 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (106 ms) 2024-07-28T19:48:20.616 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-07-28T19:48:20.616 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:20.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:20 smithi100 bash[80199]: cluster 2024-07-28T19:48:19.412592+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-07-28T19:48:20.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:20 smithi100 bash[80199]: cluster 2024-07-28T19:48:20.275169+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-07-28T19:48:20.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:20 smithi100 bash[82644]: cluster 2024-07-28T19:48:19.412592+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-07-28T19:48:20.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:20 smithi100 bash[82644]: cluster 2024-07-28T19:48:20.275169+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-07-28T19:48:20.758 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 8376263~12345 2024-07-28T19:48:20.758 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 8388608~4194304 2024-07-28T19:48:20.758 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 12582912~54321 2024-07-28T19:48:20.770 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (154 ms) 2024-07-28T19:48:20.770 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-07-28T19:48:20.770 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:48:20.770 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-07-28T19:48:20.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:20 smithi138 bash[67894]: cluster 2024-07-28T19:48:19.412592+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-07-28T19:48:20.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:20 smithi138 bash[67894]: cluster 2024-07-28T19:48:20.275169+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-07-28T19:48:21.428 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 11 tests from DiffIterateTest/0 (56449 ms total) 2024-07-28T19:48:21.428 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:48:21.428 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 11 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-07-28T19:48:21.501 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-07-28T19:48:21.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:21 smithi100 bash[80199]: cluster 2024-07-28T19:48:20.052967+0000 mgr.y (mgr.25358) 472 : cluster [DBG] pgmap v987: 105 pgs: 105 active+clean; 481 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.6 MiB/s wr, 339 op/s 2024-07-28T19:48:21.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:21 smithi100 bash[82644]: cluster 2024-07-28T19:48:20.052967+0000 mgr.y (mgr.25358) 472 : cluster [DBG] pgmap v987: 105 pgs: 105 active+clean; 481 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.6 MiB/s wr, 339 op/s 2024-07-28T19:48:21.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:21 smithi138 bash[67894]: cluster 2024-07-28T19:48:20.052967+0000 mgr.y (mgr.25358) 472 : cluster [DBG] pgmap v987: 105 pgs: 105 active+clean; 481 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.6 MiB/s wr, 339 op/s 2024-07-28T19:48:22.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:22 smithi100 bash[80199]: cluster 2024-07-28T19:48:21.425085+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-07-28T19:48:22.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:22 smithi100 bash[82644]: cluster 2024-07-28T19:48:21.425085+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-07-28T19:48:22.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:22 smithi138 bash[67894]: cluster 2024-07-28T19:48:21.425085+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-07-28T19:48:22.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:48:22 smithi138 bash[60615]: ts=2024-07-28T19:48:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:48:23.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:48:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:48:22] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.43.0" 2024-07-28T19:48:23.475 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[80199]: cluster 2024-07-28T19:48:22.053558+0000 mgr.y (mgr.25358) 473 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:48:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[80199]: cluster 2024-07-28T19:48:22.461618+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-07-28T19:48:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[80199]: audit 2024-07-28T19:48:22.466292+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.100:0/2629345981' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:48:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[80199]: audit 2024-07-28T19:48:22.466957+0000 mon.a (mon.0) 1457 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:48:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[80199]: audit 2024-07-28T19:48:22.644234+0000 mon.a (mon.0) 1458 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:23.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[82644]: cluster 2024-07-28T19:48:22.053558+0000 mgr.y (mgr.25358) 473 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:48:23.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[82644]: cluster 2024-07-28T19:48:22.461618+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-07-28T19:48:23.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[82644]: audit 2024-07-28T19:48:22.466292+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.100:0/2629345981' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:48:23.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[82644]: audit 2024-07-28T19:48:22.466957+0000 mon.a (mon.0) 1457 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:48:23.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:23 smithi100 bash[82644]: audit 2024-07-28T19:48:22.644234+0000 mon.a (mon.0) 1458 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:23 smithi138 bash[67894]: cluster 2024-07-28T19:48:22.053558+0000 mgr.y (mgr.25358) 473 : cluster [DBG] pgmap v990: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:48:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:23 smithi138 bash[67894]: cluster 2024-07-28T19:48:22.461618+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-07-28T19:48:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:23 smithi138 bash[67894]: audit 2024-07-28T19:48:22.466292+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.100:0/2629345981' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:48:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:23 smithi138 bash[67894]: audit 2024-07-28T19:48:22.466957+0000 mon.a (mon.0) 1457 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-07-28T19:48:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:23 smithi138 bash[67894]: audit 2024-07-28T19:48:22.644234+0000 mon.a (mon.0) 1458 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:24.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:24 smithi100 bash[80199]: audit 2024-07-28T19:48:23.459802+0000 mon.a (mon.0) 1459 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:48:24.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:24 smithi100 bash[80199]: cluster 2024-07-28T19:48:23.461185+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-07-28T19:48:24.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:24 smithi100 bash[80199]: cluster 2024-07-28T19:48:24.054376+0000 mgr.y (mgr.25358) 474 : cluster [DBG] pgmap v993: 105 pgs: 21 unknown, 84 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:48:24.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:24 smithi100 bash[82644]: audit 2024-07-28T19:48:23.459802+0000 mon.a (mon.0) 1459 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:48:24.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:24 smithi100 bash[82644]: cluster 2024-07-28T19:48:23.461185+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-07-28T19:48:24.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:24 smithi100 bash[82644]: cluster 2024-07-28T19:48:24.054376+0000 mgr.y (mgr.25358) 474 : cluster [DBG] pgmap v993: 105 pgs: 21 unknown, 84 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:48:24.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:24 smithi138 bash[67894]: audit 2024-07-28T19:48:23.459802+0000 mon.a (mon.0) 1459 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi100-68802-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-07-28T19:48:24.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:24 smithi138 bash[67894]: cluster 2024-07-28T19:48:23.461185+0000 mon.a (mon.0) 1460 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-07-28T19:48:24.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:24 smithi138 bash[67894]: cluster 2024-07-28T19:48:24.054376+0000 mgr.y (mgr.25358) 474 : cluster [DBG] pgmap v993: 105 pgs: 21 unknown, 84 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:48:25.539 INFO:tasks.workunit.client.0.smithi100.stdout: wrote [806663~37207,2378202~75507,2849525~5606,6313521~27843,8889917~100169,11711921~70375,12486380~46382,18991298~8194,19216148~63715,19956583~49038] 2024-07-28T19:48:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:25 smithi100 bash[80199]: cluster 2024-07-28T19:48:24.485761+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-07-28T19:48:25.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:25 smithi100 bash[82644]: cluster 2024-07-28T19:48:24.485761+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-07-28T19:48:25.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:25 smithi138 bash[67894]: cluster 2024-07-28T19:48:24.485761+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-07-28T19:48:26.534 INFO:tasks.workunit.client.0.smithi100.stdout: wrote [0~8388608,12582912~8388608] 2024-07-28T19:48:26.556 INFO:tasks.workunit.client.0.smithi100.stdout: diff was [0~8388608,12582912~8388608] 2024-07-28T19:48:26.565 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5063 ms) 2024-07-28T19:48:26.565 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-07-28T19:48:26.565 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:26.619 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:48:26.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:26 smithi138 bash[67894]: cluster 2024-07-28T19:48:25.491527+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-07-28T19:48:26.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:26 smithi138 bash[67894]: cluster 2024-07-28T19:48:26.055093+0000 mgr.y (mgr.25358) 475 : cluster [DBG] pgmap v996: 105 pgs: 4 unknown, 101 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:48:26.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:26 smithi100 bash[80199]: cluster 2024-07-28T19:48:25.491527+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-07-28T19:48:27.009 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:26 smithi100 bash[80199]: cluster 2024-07-28T19:48:26.055093+0000 mgr.y (mgr.25358) 475 : cluster [DBG] pgmap v996: 105 pgs: 4 unknown, 101 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:48:27.009 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:26 smithi100 bash[82644]: cluster 2024-07-28T19:48:25.491527+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-07-28T19:48:27.010 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:26 smithi100 bash[82644]: cluster 2024-07-28T19:48:26.055093+0000 mgr.y (mgr.25358) 475 : cluster [DBG] pgmap v996: 105 pgs: 4 unknown, 101 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-07-28T19:48:27.557 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:48:27.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:27 smithi138 bash[67894]: cluster 2024-07-28T19:48:26.512555+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-07-28T19:48:27.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:27 smithi100 bash[80199]: cluster 2024-07-28T19:48:26.512555+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-07-28T19:48:27.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:27 smithi100 bash[82644]: cluster 2024-07-28T19:48:26.512555+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-07-28T19:48:28.581 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:48:28.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:28 smithi138 bash[67894]: cluster 2024-07-28T19:48:27.523955+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-07-28T19:48:28.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:28 smithi138 bash[67894]: cluster 2024-07-28T19:48:28.055861+0000 mgr.y (mgr.25358) 476 : cluster [DBG] pgmap v999: 105 pgs: 105 active+clean; 4.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s wr, 0 op/s 2024-07-28T19:48:28.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:28 smithi100 bash[80199]: cluster 2024-07-28T19:48:27.523955+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-07-28T19:48:28.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:28 smithi100 bash[80199]: cluster 2024-07-28T19:48:28.055861+0000 mgr.y (mgr.25358) 476 : cluster [DBG] pgmap v999: 105 pgs: 105 active+clean; 4.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s wr, 0 op/s 2024-07-28T19:48:28.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:28 smithi100 bash[82644]: cluster 2024-07-28T19:48:27.523955+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-07-28T19:48:28.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:28 smithi100 bash[82644]: cluster 2024-07-28T19:48:28.055861+0000 mgr.y (mgr.25358) 476 : cluster [DBG] pgmap v999: 105 pgs: 105 active+clean; 4.3 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s wr, 0 op/s 2024-07-28T19:48:29.581 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:48:29.584 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (3019 ms) 2024-07-28T19:48:29.584 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-07-28T19:48:29.584 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:48:29.704 INFO:tasks.workunit.client.0.smithi100.stdout: i=0 exists [12328929~8129382,83290451~5366277,104063570~1338576,119386509~396264,209645636~5297665,245720424~3374776,284182108~603009,310373251~285452,364478443~6538099,396602758~5920926] wrote [12328929~8129382,83290451~5366277,104063570~1338576,119386509~396264,209645636~5297665,245720424~3374776,284182108~603009,310373251~285452,364478443~6538099,396602758~5920926] 2024-07-28T19:48:29.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:29 smithi138 bash[67894]: cluster 2024-07-28T19:48:28.535864+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-07-28T19:48:29.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:29 smithi100 bash[80199]: cluster 2024-07-28T19:48:28.535864+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-07-28T19:48:29.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:29 smithi100 bash[82644]: cluster 2024-07-28T19:48:28.535864+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-07-28T19:48:30.643 INFO:tasks.workunit.client.0.smithi100.stdout: i=1 exists [3608221~1399367,12328929~8129382,72308189~7369652,83290451~5366277,104063570~1338576,107704558~5083566,119386509~396264,131543974~5680510,146895792~8188149,192677042~2621234,209645636~5297665,245720424~3374776,266511913~1225621,284182108~603009,310373251~285452,330437676~5591169,344509486~3602666,364478443~6538099,396602758~5920926] wrote [3608221~1399367,72308189~7369652,107704558~5083566,131543974~5680510,146895792~8188149,192677042~2621234,266511913~1225621,330437676~5591169,344509486~3602666] 2024-07-28T19:48:30.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:30 smithi138 bash[67894]: cluster 2024-07-28T19:48:29.543087+0000 mon.a (mon.0) 1466 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-07-28T19:48:30.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:30 smithi138 bash[67894]: cluster 2024-07-28T19:48:30.056577+0000 mgr.y (mgr.25358) 477 : cluster [DBG] pgmap v1002: 105 pgs: 105 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 228 KiB/s wr, 50 op/s 2024-07-28T19:48:30.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:30 smithi100 bash[80199]: cluster 2024-07-28T19:48:29.543087+0000 mon.a (mon.0) 1466 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-07-28T19:48:30.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:30 smithi100 bash[80199]: cluster 2024-07-28T19:48:30.056577+0000 mgr.y (mgr.25358) 477 : cluster [DBG] pgmap v1002: 105 pgs: 105 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 228 KiB/s wr, 50 op/s 2024-07-28T19:48:30.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:30 smithi100 bash[82644]: cluster 2024-07-28T19:48:29.543087+0000 mon.a (mon.0) 1466 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-07-28T19:48:30.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:30 smithi100 bash[82644]: cluster 2024-07-28T19:48:30.056577+0000 mgr.y (mgr.25358) 477 : cluster [DBG] pgmap v1002: 105 pgs: 105 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 228 KiB/s wr, 50 op/s 2024-07-28T19:48:31.660 INFO:tasks.workunit.client.0.smithi100.stdout: i=2 exists [3608221~1399367,12328929~8129382,34326293~2958659,72308189~7369652,80793283~7863445,104063570~1338576,107704558~5083566,119386509~396264,131543974~7552349,146895792~8188149,192677042~9914726,209645636~5297665,245720424~3374776,260898010~1518719,266511913~1225621,284182108~603009,293053434~7806143,310373251~285452,319801276~1574638,326224990~2628808,330437676~5787507,344509486~8912882,364478443~6538099,396602758~5920926] wrote [34326293~2958659,80793283~7544225,134804158~4292165,194542239~8049529,260898010~1518719,293053434~7806143,319801276~1574638,326224990~2628808,334568655~1656528,347158495~6263873] 2024-07-28T19:48:31.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:31 smithi138 bash[67894]: cluster 2024-07-28T19:48:30.553278+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-07-28T19:48:31.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:31 smithi100 bash[80199]: cluster 2024-07-28T19:48:30.553278+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-07-28T19:48:31.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:31 smithi100 bash[82644]: cluster 2024-07-28T19:48:30.553278+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-07-28T19:48:32.565 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:48:32 smithi138 bash[60615]: ts=2024-07-28T19:48:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:48:32.696 INFO:tasks.workunit.client.0.smithi100.stdout: i=3 exists [314567~6537458,12328929~8129382,34326293~2958659,40022936~2964276,72308189~7369652,80793283~7863445,93295994~4538342,104063570~1338576,105749856~769930,107704558~5083566,119386509~396264,131543974~7552349,146895792~8188149,192677042~9914726,209645636~5297665,223765796~6795277,245720424~3374776,260898010~1518719,266511913~1225621,284182108~603009,293053434~7806143,310373251~285452,313117746~6406718,319801276~1574638,326224990~2628808,330437676~5787507,344509486~8912882,364478443~6538099,379958063~16208281,396602758~5920926,407669005~5638434] wrote [314567~6537458,40022936~2964276,93295994~4538342,105749856~769930,223765796~6795277,313117746~6406718,379958063~16208281,407669005~5638434] 2024-07-28T19:48:32.861 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:32 smithi100 bash[82644]: cluster 2024-07-28T19:48:31.562069+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-07-28T19:48:32.861 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:32 smithi100 bash[82644]: cluster 2024-07-28T19:48:32.057247+0000 mgr.y (mgr.25358) 478 : cluster [DBG] pgmap v1005: 105 pgs: 105 active+clean; 36 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.5 MiB/s wr, 110 op/s 2024-07-28T19:48:32.861 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:32 smithi100 bash[80199]: cluster 2024-07-28T19:48:31.562069+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-07-28T19:48:32.862 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:32 smithi100 bash[80199]: cluster 2024-07-28T19:48:32.057247+0000 mgr.y (mgr.25358) 478 : cluster [DBG] pgmap v1005: 105 pgs: 105 active+clean; 36 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.5 MiB/s wr, 110 op/s 2024-07-28T19:48:33.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:32 smithi138 bash[67894]: cluster 2024-07-28T19:48:31.562069+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-07-28T19:48:33.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:32 smithi138 bash[67894]: cluster 2024-07-28T19:48:32.057247+0000 mgr.y (mgr.25358) 478 : cluster [DBG] pgmap v1005: 105 pgs: 105 active+clean; 36 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.5 MiB/s wr, 110 op/s 2024-07-28T19:48:33.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:48:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:48:32] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.43.0" 2024-07-28T19:48:33.649 INFO:tasks.workunit.client.0.smithi100.stdout: i=4 exists [314567~6537458,11775447~8682864,34326293~2958659,37377732~8020345,72308189~7369652,80793283~7863445,93295994~10047314,104063570~1338576,105749856~769930,107704558~5083566,119386509~396264,131543974~7552349,146895792~8188149,192677042~9914726,209645636~5297665,219069528~2956354,223765796~6795277,232793013~6707022,245720424~3374776,260898010~1518719,266511913~1225621,284182108~603009,293053434~7806143,310373251~285452,313117746~6406718,319801276~1574638,326224990~2628808,328894437~7330746,344509486~8912882,364478443~6538099,379958063~16208281,396602758~5920926,407669005~5638434] wrote [11775447~1738401,37377732~8020345,97830517~5512791,219069528~2956354,223838911~737559,232793013~6707022,328894437~4107731,381108017~25891] 2024-07-28T19:48:33.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:33 smithi100 bash[80199]: cluster 2024-07-28T19:48:32.573217+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-07-28T19:48:33.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:33 smithi100 bash[82644]: cluster 2024-07-28T19:48:32.573217+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-07-28T19:48:34.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:33 smithi138 bash[67894]: cluster 2024-07-28T19:48:32.573217+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-07-28T19:48:34.681 INFO:tasks.workunit.client.0.smithi100.stdout: i=5 exists [314567~6537458,11775447~8682864,34326293~2958659,37377732~10866211,72274857~7402984,80793283~7863445,93295994~10047314,104063570~1338576,105749856~769930,107704558~5083566,119386509~396264,131543974~7552349,146895792~8188149,156958167~5074571,192677042~9914726,209645636~5297665,219069528~2956354,223765796~6795277,232793013~6707022,245720424~3374776,260699648~7447225,271653584~3746939,284182108~603009,293053434~7806143,307978973~3266280,313117746~12627889,326224990~2628808,328894437~7330746,339838705~1326315,344509486~8912882,364478443~6538099,379958063~16208281,396602758~5920926,406145797~7161642] wrote [43503912~4740031,72274857~5495230,156958167~5074571,260699648~7447225,271653584~3746939,307978973~3266280,319468507~6277128,339838705~1326315,383980355~1129745,406145797~3298928] 2024-07-28T19:48:34.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:34 smithi100 bash[80199]: cluster 2024-07-28T19:48:33.572938+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-07-28T19:48:34.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:34 smithi100 bash[80199]: audit 2024-07-28T19:48:33.798322+0000 mon.a (mon.0) 1471 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:34.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:34 smithi100 bash[80199]: cluster 2024-07-28T19:48:34.058267+0000 mgr.y (mgr.25358) 479 : cluster [DBG] pgmap v1008: 105 pgs: 105 active+clean; 148 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 24 MiB/s wr, 317 op/s 2024-07-28T19:48:34.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:34 smithi100 bash[82644]: cluster 2024-07-28T19:48:33.572938+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-07-28T19:48:35.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:34 smithi100 bash[82644]: audit 2024-07-28T19:48:33.798322+0000 mon.a (mon.0) 1471 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:35.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:34 smithi100 bash[82644]: cluster 2024-07-28T19:48:34.058267+0000 mgr.y (mgr.25358) 479 : cluster [DBG] pgmap v1008: 105 pgs: 105 active+clean; 148 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 24 MiB/s wr, 317 op/s 2024-07-28T19:48:35.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:34 smithi138 bash[67894]: cluster 2024-07-28T19:48:33.572938+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-07-28T19:48:35.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:34 smithi138 bash[67894]: audit 2024-07-28T19:48:33.798322+0000 mon.a (mon.0) 1471 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:35.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:34 smithi138 bash[67894]: cluster 2024-07-28T19:48:34.058267+0000 mgr.y (mgr.25358) 479 : cluster [DBG] pgmap v1008: 105 pgs: 105 active+clean; 148 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 24 MiB/s wr, 317 op/s 2024-07-28T19:48:35.357 INFO:tasks.workunit.client.0.smithi100.stdout: i=6 exists [314567~7973606,11775447~8682864,34326293~2958659,37377732~10866211,72274857~7402984,80793283~7863445,93295994~13498160,107704558~5083566,119169949~5729074,131543974~7552349,146895792~8188149,156958167~5074571,192677042~9914726,209645636~5297665,219069528~2956354,223765796~6795277,232793013~6707022,245720424~3374776,260699648~7447225,271653584~3746939,281301533~4758427,293053434~7806143,307978973~3266280,313117746~12627889,326224990~2628808,328894437~7330746,339838705~1326315,344509486~8912882,364478443~6538099,379958063~16208281,396602758~16704681] wrote [2900706~5387467,82196731~1185217,101702218~5091936,119169949~5729074,193214651~2952535,234415165~1574398,281301533~4758427,320585456~1392836,393931164~981715,399163127~7440424] 2024-07-28T19:48:35.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:35 smithi100 bash[80199]: cluster 2024-07-28T19:48:34.591752+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-07-28T19:48:35.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:35 smithi100 bash[80199]: cluster 2024-07-28T19:48:35.279179+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-07-28T19:48:35.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:35 smithi100 bash[82644]: cluster 2024-07-28T19:48:34.591752+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-07-28T19:48:35.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:35 smithi100 bash[82644]: cluster 2024-07-28T19:48:35.279179+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-07-28T19:48:36.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:35 smithi138 bash[67894]: cluster 2024-07-28T19:48:34.591752+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-07-28T19:48:36.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:35 smithi138 bash[67894]: cluster 2024-07-28T19:48:35.279179+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-07-28T19:48:36.387 INFO:tasks.workunit.client.0.smithi100.stdout: i=7 exists [314567~7973606,11775447~8682864,34326293~2958659,37377732~10866211,72274857~7402984,80793283~9648086,93295994~13498160,107704558~5083566,119169949~5729074,131543974~7552349,139749757~15334184,156958167~5074571,181316596~3262686,192677042~9914726,209645636~5297665,219069528~11491545,232793013~6707022,245720424~3374776,260699648~7447225,271653584~3746939,281301533~4758427,293053434~7806143,307978973~3266280,313117746~12627889,326224990~2628808,328894437~7330746,339838705~1326315,344509486~8912882,364478443~6538099,372831574~2409333,379958063~16208281,396602758~16704681] wrote [12811728~5011202,85481414~4959955,139749757~8991950,181316596~3262686,220635987~6257990,372831574~2409333,381632057~1458540,406441327~771785] 2024-07-28T19:48:37.379 INFO:tasks.workunit.client.0.smithi100.stdout: i=8 exists [314567~7973606,9785024~10673287,34326293~2958659,37377732~10866211,72274857~7402984,80793283~9648086,93295994~13498160,107704558~5083566,119169949~5729074,131543974~7552349,139749757~15334184,156958167~5074571,162271887~78592,181316596~3262686,192677042~9914726,209645636~5297665,219069528~11491545,232793013~8469290,245720424~3374776,260699648~7447225,271653584~6587855,281301533~4758427,293053434~7806143,307978973~3266280,313117746~12627889,326224990~2628808,328894437~7330746,339838705~1326315,344509486~14040043,364478443~6538099,372831574~2409333,379958063~16208281,396602758~16704681] wrote [9785024~4258758,16068438~918337,99221951~1734195,147825170~639349,162271887~78592,237292571~3969732,272549201~5692238,315924016~7554218,352824056~5725473,399345003~7032401] 2024-07-28T19:48:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:37 smithi138 bash[67894]: cluster 2024-07-28T19:48:36.058996+0000 mgr.y (mgr.25358) 480 : cluster [DBG] pgmap v1011: 105 pgs: 105 active+clean; 280 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 162 KiB/s rd, 46 MiB/s wr, 274 op/s 2024-07-28T19:48:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:37 smithi138 bash[67894]: cluster 2024-07-28T19:48:36.283485+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-07-28T19:48:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:37 smithi138 bash[67894]: audit 2024-07-28T19:48:37.168632+0000 mon.a (mon.0) 1475 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:48:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:37 smithi100 bash[80199]: cluster 2024-07-28T19:48:36.058996+0000 mgr.y (mgr.25358) 480 : cluster [DBG] pgmap v1011: 105 pgs: 105 active+clean; 280 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 162 KiB/s rd, 46 MiB/s wr, 274 op/s 2024-07-28T19:48:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:37 smithi100 bash[80199]: cluster 2024-07-28T19:48:36.283485+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-07-28T19:48:37.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:37 smithi100 bash[80199]: audit 2024-07-28T19:48:37.168632+0000 mon.a (mon.0) 1475 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:48:37.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:37 smithi100 bash[82644]: cluster 2024-07-28T19:48:36.058996+0000 mgr.y (mgr.25358) 480 : cluster [DBG] pgmap v1011: 105 pgs: 105 active+clean; 280 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 162 KiB/s rd, 46 MiB/s wr, 274 op/s 2024-07-28T19:48:37.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:37 smithi100 bash[82644]: cluster 2024-07-28T19:48:36.283485+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-07-28T19:48:37.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:37 smithi100 bash[82644]: audit 2024-07-28T19:48:37.168632+0000 mon.a (mon.0) 1475 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:48:38.403 INFO:tasks.workunit.client.0.smithi100.stdout: i=9 exists [314567~7973606,9785024~10673287,34326293~2958659,37377732~10866211,50670156~6038659,72274857~7402984,80793283~9648086,93295994~13498160,107704558~5083566,113146588~356941,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~5074571,162271887~78592,170889493~4508684,181316596~4240992,192677042~9914726,209645636~20915437,232666125~8596178,245047366~4047834,260699648~7447225,271653584~6587855,281301533~4758427,293053434~7806143,307978973~3266280,313117746~12627889,326224990~2628808,328894437~7330746,339838705~4184797,344509486~14040043,364478443~6538099,372831574~2409333,379958063~16208281,396602758~16704681] wrote [50670156~6038659,113146588~356941,125268482~4999745,170889493~4508684,184084802~1472786,213363728~6758922,232666125~6154469,245047366~3931091,263808689~3415297,341071415~2952087] 2024-07-28T19:48:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:38 smithi138 bash[67894]: cluster 2024-07-28T19:48:37.288938+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-07-28T19:48:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:38 smithi138 bash[67894]: audit 2024-07-28T19:48:37.583946+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:38 smithi138 bash[67894]: audit 2024-07-28T19:48:37.589859+0000 mon.a (mon.0) 1478 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:38 smithi138 bash[67894]: audit 2024-07-28T19:48:37.644412+0000 mon.a (mon.0) 1479 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:38.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:38 smithi100 bash[80199]: cluster 2024-07-28T19:48:37.288938+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-07-28T19:48:38.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:38 smithi100 bash[80199]: audit 2024-07-28T19:48:37.583946+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:38.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:38 smithi100 bash[80199]: audit 2024-07-28T19:48:37.589859+0000 mon.a (mon.0) 1478 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:38.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:38 smithi100 bash[80199]: audit 2024-07-28T19:48:37.644412+0000 mon.a (mon.0) 1479 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:38.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:38 smithi100 bash[82644]: cluster 2024-07-28T19:48:37.288938+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-07-28T19:48:38.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:38 smithi100 bash[82644]: audit 2024-07-28T19:48:37.583946+0000 mon.a (mon.0) 1477 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:38.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:38 smithi100 bash[82644]: audit 2024-07-28T19:48:37.589859+0000 mon.a (mon.0) 1478 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:38.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:38 smithi100 bash[82644]: audit 2024-07-28T19:48:37.644412+0000 mon.a (mon.0) 1479 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:39.426 INFO:tasks.workunit.client.0.smithi100.stdout: i=10 exists [314567~20143744,34326293~2958659,37377732~23449737,72274857~7402984,80793283~9648086,93295994~13498160,107704558~5083566,113146588~356941,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~10691112,170889493~4508684,181316596~4240992,192677042~9914726,209645636~20915437,232666125~8596178,245047366~11679532,260699648~7447225,271653584~6587855,281301533~4758427,293053434~7806143,307060655~4184598,313117746~12627889,326224990~2628808,328894437~7330746,339838705~18710824,364478443~6538099,372831574~2409333,379958063~16208281,396602758~16704681] wrote [3898608~5904798,46492494~5494458,54243843~6583626,84263778~5919772,161541979~6107300,248575723~8151175,307060655~1117706,343320415~4516896,405726629~1151940] 2024-07-28T19:48:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:39 smithi138 bash[67894]: cluster 2024-07-28T19:48:38.059861+0000 mgr.y (mgr.25358) 481 : cluster [DBG] pgmap v1014: 105 pgs: 105 active+clean; 314 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 34 MiB/s wr, 19 op/s 2024-07-28T19:48:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:39 smithi138 bash[67894]: cluster 2024-07-28T19:48:38.301057+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-07-28T19:48:39.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:39 smithi100 bash[80199]: cluster 2024-07-28T19:48:38.059861+0000 mgr.y (mgr.25358) 481 : cluster [DBG] pgmap v1014: 105 pgs: 105 active+clean; 314 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 34 MiB/s wr, 19 op/s 2024-07-28T19:48:39.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:39 smithi100 bash[80199]: cluster 2024-07-28T19:48:38.301057+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-07-28T19:48:39.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:39 smithi100 bash[82644]: cluster 2024-07-28T19:48:38.059861+0000 mgr.y (mgr.25358) 481 : cluster [DBG] pgmap v1014: 105 pgs: 105 active+clean; 314 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 34 MiB/s wr, 19 op/s 2024-07-28T19:48:39.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:39 smithi100 bash[82644]: cluster 2024-07-28T19:48:38.301057+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-07-28T19:48:40.459 INFO:tasks.workunit.client.0.smithi100.stdout: i=11 exists [314567~20143744,34326293~2958659,37377732~23449737,72274857~7402984,80793283~9648086,93295994~13498160,107704558~5083566,113146588~356941,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~10691112,170889493~4508684,176876321~8681267,186415655~3160064,192677042~9914726,209645636~20915437,232666125~8596178,245047366~11679532,260699648~7447225,268409418~9832021,281301533~4758427,293053434~10712135,307060655~4184598,313117746~12627889,326224990~10000193,339838705~18710824,360389162~1893754,364478443~6538099,372831574~2409333,379958063~16208281,396602758~16704681] wrote [49419800~8110271,97292445~8072717,176876321~4998751,186415655~3160064,268409418~5743953,297471669~6293900,326944005~6347139,347195225~8695037,360389162~1893754] 2024-07-28T19:48:40.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:40 smithi100 bash[80199]: cluster 2024-07-28T19:48:39.315798+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-07-28T19:48:40.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:40 smithi100 bash[82644]: cluster 2024-07-28T19:48:39.315798+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-07-28T19:48:40.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:40 smithi138 bash[67894]: cluster 2024-07-28T19:48:39.315798+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-07-28T19:48:41.443 INFO:tasks.workunit.client.0.smithi100.stdout: i=12 exists [314567~20143744,34326293~2958659,37377732~23449737,72274857~7402984,80793283~9648086,93295994~13498160,107704558~5083566,113146588~356941,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~10691112,170889493~4508684,176876321~8681267,186415655~3160064,192677042~9914726,209645636~20915437,232666125~8596178,245047366~11679532,260699648~7447225,268409418~9832021,281301533~4758427,293053434~10712135,307060655~4184598,313117746~12627889,326224990~10000193,339838705~18710824,360389162~1893754,364478443~11380377,376435486~19730858,396602758~16704681] wrote [3100670~2742473,9829807~4784764,113461464~11009,212928587~6375153,239362981~1566931,295975245~6819423,368338988~7519832,376435486~5338253,397559289~1038712] 2024-07-28T19:48:41.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:41 smithi100 bash[80199]: cluster 2024-07-28T19:48:40.060805+0000 mgr.y (mgr.25358) 482 : cluster [DBG] pgmap v1017: 105 pgs: 105 active+clean; 428 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 36 MiB/s wr, 304 op/s 2024-07-28T19:48:41.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:41 smithi100 bash[80199]: cluster 2024-07-28T19:48:40.329199+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-07-28T19:48:41.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:41 smithi100 bash[82644]: cluster 2024-07-28T19:48:40.060805+0000 mgr.y (mgr.25358) 482 : cluster [DBG] pgmap v1017: 105 pgs: 105 active+clean; 428 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 36 MiB/s wr, 304 op/s 2024-07-28T19:48:41.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:41 smithi100 bash[82644]: cluster 2024-07-28T19:48:40.329199+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-07-28T19:48:41.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:41 smithi138 bash[67894]: cluster 2024-07-28T19:48:40.060805+0000 mgr.y (mgr.25358) 482 : cluster [DBG] pgmap v1017: 105 pgs: 105 active+clean; 428 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 190 KiB/s rd, 36 MiB/s wr, 304 op/s 2024-07-28T19:48:41.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:41 smithi138 bash[67894]: cluster 2024-07-28T19:48:40.329199+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-07-28T19:48:42.441 INFO:tasks.workunit.client.0.smithi100.stdout: i=13 exists [314567~20143744,34326293~2958659,37377732~23449737,72274857~7402984,80793283~9648086,93295994~13498160,107704558~5083566,113146588~356941,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~10691112,170889493~4508684,176876321~8681267,186415655~3160064,192677042~9914726,204202221~3031501,209645636~20915437,232666125~8596178,245047366~11679532,260699648~7447225,268409418~9832021,281301533~8111891,293053434~10712135,307060655~4184598,313117746~12627889,326224990~10000193,339838705~18710824,360389162~1893754,364478443~11380377,376435486~19972617,396602758~16704681] wrote [57172256~1143460,148836726~1930042,161053891~3783610,204202221~3031501,248605268~3658697,283793676~5619748,314353050~7398758,388504244~7903859] 2024-07-28T19:48:42.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:42 smithi100 bash[80199]: cluster 2024-07-28T19:48:41.327598+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-07-28T19:48:42.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:42 smithi100 bash[82644]: cluster 2024-07-28T19:48:41.327598+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-07-28T19:48:42.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:48:42 smithi138 bash[60615]: ts=2024-07-28T19:48:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:48:42.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:42 smithi138 bash[67894]: cluster 2024-07-28T19:48:41.327598+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-07-28T19:48:43.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:48:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:48:42] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2024-07-28T19:48:43.422 INFO:tasks.workunit.client.0.smithi100.stdout: i=14 exists [314567~20143744,34326293~2958659,37377732~23449737,72274857~7452049,80793283~9648086,93295994~13498160,107704558~5083566,113146588~356941,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~10691112,170889493~4508684,176876321~8681267,186415655~3160064,192677042~9914726,204202221~3031501,209645636~21959583,232666125~8596178,245047366~11679532,260699648~7447225,268409418~9832021,281301533~8111891,293053434~11036119,307060655~4184598,313117746~12634299,326224990~32324539,360389162~1893754,364478443~11380377,376435486~19972617,396602758~16704681] wrote [75681524~4045382,210216879~1654911,229605629~1999590,302831957~1257596,319154031~6598014,333362100~987965,334849921~7196218,388531550~4111473,402615711~1882408,407001288~229041] 2024-07-28T19:48:43.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:43 smithi100 bash[80199]: cluster 2024-07-28T19:48:42.061502+0000 mgr.y (mgr.25358) 483 : cluster [DBG] pgmap v1020: 105 pgs: 105 active+clean; 487 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 42 MiB/s wr, 310 op/s 2024-07-28T19:48:43.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:43 smithi100 bash[80199]: cluster 2024-07-28T19:48:42.345270+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-07-28T19:48:43.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:43 smithi100 bash[80199]: audit 2024-07-28T19:48:42.860441+0000 mon.a (mon.0) 1485 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:43.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:43 smithi100 bash[80199]: audit 2024-07-28T19:48:42.866997+0000 mon.a (mon.0) 1486 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:43.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:43 smithi100 bash[82644]: cluster 2024-07-28T19:48:42.061502+0000 mgr.y (mgr.25358) 483 : cluster [DBG] pgmap v1020: 105 pgs: 105 active+clean; 487 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 42 MiB/s wr, 310 op/s 2024-07-28T19:48:43.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:43 smithi100 bash[82644]: cluster 2024-07-28T19:48:42.345270+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-07-28T19:48:43.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:43 smithi100 bash[82644]: audit 2024-07-28T19:48:42.860441+0000 mon.a (mon.0) 1485 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:43.751 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:43 smithi100 bash[82644]: audit 2024-07-28T19:48:42.866997+0000 mon.a (mon.0) 1486 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:43.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:43 smithi138 bash[67894]: cluster 2024-07-28T19:48:42.061502+0000 mgr.y (mgr.25358) 483 : cluster [DBG] pgmap v1020: 105 pgs: 105 active+clean; 487 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 42 MiB/s wr, 310 op/s 2024-07-28T19:48:43.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:43 smithi138 bash[67894]: cluster 2024-07-28T19:48:42.345270+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-07-28T19:48:43.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:43 smithi138 bash[67894]: audit 2024-07-28T19:48:42.860441+0000 mon.a (mon.0) 1485 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:43.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:43 smithi138 bash[67894]: audit 2024-07-28T19:48:42.866997+0000 mon.a (mon.0) 1486 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.538 INFO:tasks.workunit.client.0.smithi100.stdout: i=15 exists [314567~23939772,32898393~4386559,37377732~23449737,68645443~21795926,93295994~13498160,107704558~5083566,113146588~356941,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~10691112,168669194~6728983,176876321~8681267,186415655~3160064,192677042~9914726,204202221~3031501,209645636~21959583,232666125~8596178,245047366~14309696,260699648~7447225,268409418~9832021,281301533~8111891,293053434~11036119,304539681~1773062,307060655~4184598,313117746~12634299,326224990~32324539,360389162~1893754,364478443~11380377,376278273~20129830,396602758~16704681] wrote [19253714~5000625,32898393~1438776,45005578~255183,68645443~4351026,76803209~4397666,159541102~3752173,168669194~5305870,251497651~7859411,304539681~1773062,376278273~255118] 2024-07-28T19:48:44.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[80199]: cluster 2024-07-28T19:48:43.355977+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-07-28T19:48:44.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[80199]: audit 2024-07-28T19:48:43.408579+0000 mon.a (mon.0) 1488 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[80199]: audit 2024-07-28T19:48:43.434799+0000 mon.a (mon.0) 1489 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[80199]: audit 2024-07-28T19:48:43.890243+0000 mon.a (mon.0) 1490 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:48:44.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[80199]: audit 2024-07-28T19:48:43.892197+0000 mon.a (mon.0) 1491 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:48:44.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[80199]: audit 2024-07-28T19:48:43.904562+0000 mon.a (mon.0) 1492 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[82644]: cluster 2024-07-28T19:48:43.355977+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-07-28T19:48:44.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[82644]: audit 2024-07-28T19:48:43.408579+0000 mon.a (mon.0) 1488 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[82644]: audit 2024-07-28T19:48:43.434799+0000 mon.a (mon.0) 1489 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[82644]: audit 2024-07-28T19:48:43.890243+0000 mon.a (mon.0) 1490 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:48:44.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[82644]: audit 2024-07-28T19:48:43.892197+0000 mon.a (mon.0) 1491 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:48:44.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:44 smithi100 bash[82644]: audit 2024-07-28T19:48:43.904562+0000 mon.a (mon.0) 1492 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:44 smithi138 bash[67894]: cluster 2024-07-28T19:48:43.355977+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-07-28T19:48:44.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:44 smithi138 bash[67894]: audit 2024-07-28T19:48:43.408579+0000 mon.a (mon.0) 1488 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:44 smithi138 bash[67894]: audit 2024-07-28T19:48:43.434799+0000 mon.a (mon.0) 1489 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:44.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:44 smithi138 bash[67894]: audit 2024-07-28T19:48:43.890243+0000 mon.a (mon.0) 1490 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:48:44.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:44 smithi138 bash[67894]: audit 2024-07-28T19:48:43.892197+0000 mon.a (mon.0) 1491 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:48:44.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:44 smithi138 bash[67894]: audit 2024-07-28T19:48:43.904562+0000 mon.a (mon.0) 1492 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:48:45.554 INFO:tasks.workunit.client.0.smithi100.stdout: i=16 exists [314567~23939772,32898393~4386559,37377732~23449737,68645443~21795926,91257321~15536833,107704558~7177209,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~10691112,168669194~6728983,176876321~8681267,186415655~3160064,192677042~9914726,204202221~3031501,209645636~21959583,232666125~8596178,245047366~14309696,260699648~7447225,268409418~10510519,281301533~8111891,293053434~11036119,304539681~1773062,307060655~4184598,313117746~12634299,326224990~32324539,360389162~2300535,364478443~11380377,376278273~20129830,396602758~16704681] wrote [7843709~5143495,43676112~7881285,71980470~3301261,91257321~7682986,100175859~2688754,111746019~3135748,199565965~62779,275188090~3731847,361178444~1511253,404988338~4547261] 2024-07-28T19:48:45.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:45 smithi100 bash[80199]: cluster 2024-07-28T19:48:44.062385+0000 mgr.y (mgr.25358) 484 : cluster [DBG] pgmap v1023: 105 pgs: 105 active+clean; 580 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 225 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-07-28T19:48:45.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:45 smithi100 bash[80199]: cluster 2024-07-28T19:48:44.438423+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-07-28T19:48:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:45 smithi100 bash[82644]: cluster 2024-07-28T19:48:44.062385+0000 mgr.y (mgr.25358) 484 : cluster [DBG] pgmap v1023: 105 pgs: 105 active+clean; 580 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 225 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-07-28T19:48:45.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:45 smithi100 bash[82644]: cluster 2024-07-28T19:48:44.438423+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-07-28T19:48:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:45 smithi138 bash[67894]: cluster 2024-07-28T19:48:44.062385+0000 mgr.y (mgr.25358) 484 : cluster [DBG] pgmap v1023: 105 pgs: 105 active+clean; 580 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 225 KiB/s rd, 38 MiB/s wr, 354 op/s 2024-07-28T19:48:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:45 smithi138 bash[67894]: cluster 2024-07-28T19:48:44.438423+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-07-28T19:48:46.552 INFO:tasks.workunit.client.0.smithi100.stdout: i=17 exists [314567~23939772,32898393~4386559,37377732~23449737,68645443~21795926,91257321~15536833,107704558~7570470,119169949~5729074,125268482~4999745,131543974~7552349,139749757~15334184,156958167~10691112,168669194~6728983,176876321~8681267,186415655~3160064,192677042~14556680,209645636~21959583,232666125~8596178,245047366~14309696,260699648~7447225,268409418~10510519,281301533~8111891,293053434~11036119,304539681~1773062,307060655~18691390,326224990~32324539,360389162~2300535,364478443~11380377,376278273~20129830,396602758~16704681] wrote [108676844~6598184,135567738~3500414,199594559~901244,202488065~4104620,223388511~4891389,288104499~30744,310604924~8107618,388306166~821162,404855596~1849240] 2024-07-28T19:48:46.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:46 smithi100 bash[80199]: cluster 2024-07-28T19:48:45.447318+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-07-28T19:48:46.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:46 smithi100 bash[82644]: cluster 2024-07-28T19:48:45.447318+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-07-28T19:48:46.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:46 smithi138 bash[67894]: cluster 2024-07-28T19:48:45.447318+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-07-28T19:48:47.581 INFO:tasks.workunit.client.0.smithi100.stdout: i=18 exists [314567~23939772,32898393~4386559,37377732~23449737,68645443~21795926,91257321~15536833,107704558~7570470,119141119~11127108,131543974~7552349,139749757~15334184,156958167~10691112,168669194~6728983,176876321~8681267,186415655~3160064,190133067~17100655,209645636~21959583,232666125~8596178,245047366~14309696,260699648~7447225,268409418~10510519,281301533~8111891,293053434~11036119,304474606~21277439,326224990~32324539,360389162~2300535,364478443~11380377,376278273~20129830,396602758~16704681] wrote [44135455~1225744,69476032~5943484,80960254~217826,119141119~1846364,121502475~6637636,190133067~6810744,304474606~5761182,316453481~561304,376793712~5734354,404737085~7963380] 2024-07-28T19:48:47.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:47 smithi100 bash[80199]: cluster 2024-07-28T19:48:46.063053+0000 mgr.y (mgr.25358) 485 : cluster [DBG] pgmap v1026: 105 pgs: 105 active+clean; 668 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 225 KiB/s rd, 46 MiB/s wr, 359 op/s 2024-07-28T19:48:47.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:47 smithi100 bash[80199]: cluster 2024-07-28T19:48:46.460387+0000 mon.a (mon.0) 1495 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-07-28T19:48:47.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:47 smithi100 bash[82644]: cluster 2024-07-28T19:48:46.063053+0000 mgr.y (mgr.25358) 485 : cluster [DBG] pgmap v1026: 105 pgs: 105 active+clean; 668 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 225 KiB/s rd, 46 MiB/s wr, 359 op/s 2024-07-28T19:48:47.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:47 smithi100 bash[82644]: cluster 2024-07-28T19:48:46.460387+0000 mon.a (mon.0) 1495 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-07-28T19:48:47.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:47 smithi138 bash[67894]: cluster 2024-07-28T19:48:46.063053+0000 mgr.y (mgr.25358) 485 : cluster [DBG] pgmap v1026: 105 pgs: 105 active+clean; 668 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 225 KiB/s rd, 46 MiB/s wr, 359 op/s 2024-07-28T19:48:47.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:47 smithi138 bash[67894]: cluster 2024-07-28T19:48:46.460387+0000 mon.a (mon.0) 1495 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-07-28T19:48:48.558 INFO:tasks.workunit.client.0.smithi100.stdout: i=19 exists [314567~23939772,32898393~4386559,37377732~23449737,68645443~21795926,91257321~15536833,107704558~7570470,119141119~11127108,131543974~7552349,139749757~15334184,156958167~10691112,168669194~6728983,176876321~8681267,186415655~3160064,190133067~17100655,207367575~24237644,232666125~8596178,245047366~14309696,260699648~7447225,268409418~10510519,281301533~8111891,293053434~11036119,304474606~21277439,326224990~32324539,360389162~2300535,364478443~11380377,376278273~20129830,396602758~16704681] wrote [9221989~2730601,77024977~4727128,83886738~2823658,207367575~4161072,216225932~1531801,222676502~1572438,299112480~132512,334849999~4258801,386167692~6816064] 2024-07-28T19:48:48.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:48 smithi100 bash[80199]: cluster 2024-07-28T19:48:47.466764+0000 mon.a (mon.0) 1496 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-07-28T19:48:48.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:48 smithi100 bash[80199]: cluster 2024-07-28T19:48:48.063697+0000 mgr.y (mgr.25358) 486 : cluster [DBG] pgmap v1029: 105 pgs: 105 active+clean; 690 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 28 MiB/s wr, 16 op/s 2024-07-28T19:48:48.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:48 smithi100 bash[82644]: cluster 2024-07-28T19:48:47.466764+0000 mon.a (mon.0) 1496 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-07-28T19:48:48.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:48 smithi100 bash[82644]: cluster 2024-07-28T19:48:48.063697+0000 mgr.y (mgr.25358) 486 : cluster [DBG] pgmap v1029: 105 pgs: 105 active+clean; 690 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 28 MiB/s wr, 16 op/s 2024-07-28T19:48:48.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:48 smithi138 bash[67894]: cluster 2024-07-28T19:48:47.466764+0000 mon.a (mon.0) 1496 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-07-28T19:48:48.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:48 smithi138 bash[67894]: cluster 2024-07-28T19:48:48.063697+0000 mgr.y (mgr.25358) 486 : cluster [DBG] pgmap v1029: 105 pgs: 105 active+clean; 690 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 28 MiB/s wr, 16 op/s 2024-07-28T19:48:49.510 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 1 diff [0~4194304,71303168~8388608,104857600~8388608,130023424~8388608,146800640~8388608,188743680~4194304,264241152~4194304,327155712~8388608,343932928~4194304] 2024-07-28T19:48:49.510 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,71303168~8388608,104857600~8388608,130023424~8388608,146800640~8388608,188743680~4194304,264241152~4194304,327155712~8388608,343932928~4194304] 2024-07-28T19:48:49.541 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,71303168~8388608,104857600~8388608,130023424~8388608,146800640~8388608,188743680~8388608,264241152~4194304,327155712~12582912,343932928~4194304] 2024-07-28T19:48:49.541 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 2 diff [0~4194304,33554432~4194304,71303168~16777216,104857600~8388608,130023424~8388608,146800640~8388608,188743680~12582912,260046848~8388608,289406976~8388608,318767104~16777216,343932928~12582912] 2024-07-28T19:48:49.541 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,33554432~4194304,71303168~16777216,104857600~8388608,130023424~8388608,146800640~8388608,188743680~12582912,260046848~8388608,289406976~8388608,318767104~16777216,343932928~12582912] 2024-07-28T19:48:49.545 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,33554432~4194304,71303168~20971520,104857600~8388608,130023424~12582912,146800640~8388608,188743680~16777216,260046848~8388608,289406976~12582912,318767104~20971520,343932928~12582912] 2024-07-28T19:48:49.546 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 3 diff [0~8388608,33554432~8388608,71303168~16777216,92274688~8388608,104857600~8388608,130023424~8388608,146800640~8388608,188743680~12582912,222298112~8388608,260046848~8388608,289406976~8388608,310378496~25165824,343932928~12582912,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.546 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,33554432~8388608,71303168~16777216,92274688~8388608,104857600~8388608,130023424~8388608,146800640~8388608,188743680~12582912,222298112~8388608,260046848~8388608,289406976~8388608,310378496~25165824,343932928~12582912,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.551 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,33554432~12582912,71303168~29360128,104857600~8388608,130023424~12582912,146800640~8388608,188743680~16777216,222298112~8388608,260046848~8388608,289406976~12582912,310378496~29360128,343932928~12582912,377487360~20971520,406847488~8388608] 2024-07-28T19:48:49.552 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 4 diff [0~12582912,33554432~8388608,71303168~16777216,92274688~20971520,130023424~8388608,146800640~8388608,188743680~12582912,218103808~20971520,260046848~8388608,289406976~8388608,310378496~25165824,343932928~12582912,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.552 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~8388608,71303168~16777216,92274688~20971520,130023424~8388608,146800640~8388608,188743680~12582912,218103808~20971520,260046848~8388608,289406976~8388608,310378496~25165824,343932928~12582912,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.558 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~12582912,71303168~41943040,130023424~12582912,146800640~8388608,188743680~16777216,218103808~25165824,260046848~8388608,289406976~12582912,310378496~29360128,343932928~12582912,377487360~20971520,406847488~8388608] 2024-07-28T19:48:49.558 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 5 diff [0~12582912,33554432~12582912,71303168~16777216,92274688~20971520,130023424~8388608,146800640~16777216,188743680~12582912,218103808~20971520,260046848~12582912,289406976~8388608,306184192~29360128,339738624~16777216,377487360~16777216,402653184~8388608] 2024-07-28T19:48:49.559 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~12582912,71303168~16777216,92274688~20971520,130023424~8388608,146800640~16777216,188743680~12582912,218103808~20971520,260046848~12582912,289406976~8388608,306184192~29360128,339738624~16777216,377487360~16777216,402653184~8388608] 2024-07-28T19:48:49.566 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~16777216,71303168~41943040,130023424~12582912,146800640~16777216,188743680~16777216,218103808~25165824,260046848~16777216,289406976~12582912,306184192~50331648,377487360~20971520,402653184~12582912] 2024-07-28T19:48:49.566 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 6 diff [0~12582912,33554432~12582912,71303168~16777216,92274688~20971520,117440512~8388608,130023424~8388608,146800640~16777216,188743680~12582912,218103808~20971520,260046848~12582912,281018368~16777216,306184192~29360128,339738624~16777216,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.566 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~12582912,71303168~16777216,92274688~20971520,117440512~8388608,130023424~8388608,146800640~16777216,188743680~12582912,218103808~20971520,260046848~12582912,281018368~16777216,306184192~29360128,339738624~16777216,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.574 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~16777216,71303168~41943040,117440512~8388608,130023424~12582912,146800640~16777216,188743680~16777216,218103808~25165824,260046848~16777216,281018368~20971520,306184192~50331648,377487360~37748736] 2024-07-28T19:48:49.574 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 7 diff [0~16777216,33554432~12582912,71303168~41943040,117440512~8388608,130023424~33554432,180355072~4194304,188743680~12582912,218103808~20971520,260046848~12582912,281018368~16777216,306184192~29360128,339738624~16777216,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.574 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,71303168~41943040,117440512~8388608,130023424~33554432,180355072~4194304,188743680~12582912,218103808~20971520,260046848~12582912,281018368~16777216,306184192~29360128,339738624~16777216,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.584 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~16777216,71303168~41943040,117440512~8388608,130023424~33554432,180355072~25165824,218103808~25165824,260046848~16777216,281018368~20971520,306184192~50331648,369098752~46137344] 2024-07-28T19:48:49.584 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 8 diff [0~16777216,33554432~12582912,71303168~41943040,117440512~8388608,130023424~33554432,180355072~4194304,188743680~12582912,218103808~25165824,260046848~16777216,281018368~16777216,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.584 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,71303168~41943040,117440512~8388608,130023424~33554432,180355072~4194304,188743680~12582912,218103808~25165824,260046848~16777216,281018368~16777216,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.594 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~16777216,71303168~41943040,117440512~8388608,130023424~33554432,180355072~25165824,218103808~25165824,260046848~41943040,306184192~54525952,369098752~46137344] 2024-07-28T19:48:49.594 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 9 diff [0~16777216,33554432~12582912,50331648~8388608,71303168~41943040,117440512~46137344,167772160~8388608,180355072~20971520,209715200~37748736,260046848~16777216,281018368~16777216,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.595 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,50331648~8388608,71303168~41943040,117440512~46137344,167772160~8388608,180355072~20971520,209715200~37748736,260046848~16777216,281018368~16777216,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.606 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~25165824,71303168~92274688,167772160~8388608,180355072~25165824,209715200~41943040,260046848~41943040,306184192~54525952,369098752~46137344] 2024-07-28T19:48:49.606 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 10 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~58720256,180355072~20971520,209715200~46137344,260046848~16777216,281018368~16777216,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.606 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~58720256,180355072~20971520,209715200~46137344,260046848~16777216,281018368~16777216,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.618 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~104857600,180355072~25165824,209715200~92274688,306184192~54525952,369098752~46137344] 2024-07-28T19:48:49.618 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 11 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~83886080,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.618 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~83886080,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.634 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~134217728,209715200~155189248,369098752~46137344] 2024-07-28T19:48:49.634 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 12 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~83886080,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,364904448~50331648] 2024-07-28T19:48:49.634 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~83886080,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,364904448~50331648] 2024-07-28T19:48:49.646 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~134217728,209715200~205520896] 2024-07-28T19:48:49.647 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 13 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~88080384,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,364904448~50331648] 2024-07-28T19:48:49.647 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~88080384,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,364904448~50331648] 2024-07-28T19:48:49.661 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~343932928] 2024-07-28T19:48:49.662 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 14 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~88080384,209715200~46137344,260046848~16777216,281018368~20971520,306184192~54525952,364904448~50331648] 2024-07-28T19:48:49.662 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~88080384,209715200~46137344,260046848~16777216,281018368~20971520,306184192~54525952,364904448~50331648] 2024-07-28T19:48:49.677 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~343932928] 2024-07-28T19:48:49.678 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 15 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~88080384,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.678 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~88080384,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.694 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:49.695 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 16 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~88080384,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.695 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~88080384,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.713 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:49.713 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 17 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~88080384,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.713 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~88080384,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.733 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:49.733 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 18 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.733 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:49 smithi100 bash[82644]: cluster 2024-07-28T19:48:48.484891+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-07-28T19:48:49.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:49 smithi100 bash[80199]: cluster 2024-07-28T19:48:48.484891+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-07-28T19:48:49.753 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:49.753 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to 19 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.753 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.773 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:49.774 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 2 diff [33554432~4194304,79691776~8388608,134217728~8388608,192937984~8388608,260046848~4194304,289406976~8388608,318767104~8388608,331350016~4194304,343932928~8388608] 2024-07-28T19:48:49.774 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [33554432~4194304,79691776~8388608,134217728~8388608,192937984~8388608,260046848~4194304,289406976~8388608,318767104~8388608,331350016~4194304,343932928~8388608] 2024-07-28T19:48:49.777 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [33554432~4194304,79691776~12582912,134217728~8388608,192937984~12582912,260046848~4194304,289406976~12582912,318767104~20971520,343932928~12582912] 2024-07-28T19:48:49.777 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 3 diff [0~8388608,33554432~8388608,79691776~8388608,92274688~8388608,104857600~4194304,134217728~8388608,192937984~8388608,222298112~8388608,260046848~4194304,289406976~8388608,310378496~16777216,331350016~4194304,343932928~8388608,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.777 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,33554432~8388608,79691776~8388608,92274688~8388608,104857600~4194304,134217728~8388608,192937984~8388608,222298112~8388608,260046848~4194304,289406976~8388608,310378496~16777216,331350016~4194304,343932928~8388608,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.781 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,33554432~12582912,79691776~20971520,104857600~4194304,134217728~8388608,192937984~12582912,222298112~8388608,260046848~4194304,289406976~12582912,310378496~29360128,343932928~12582912,377487360~20971520,406847488~8388608] 2024-07-28T19:48:49.781 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 4 diff [0~12582912,33554432~8388608,79691776~8388608,92274688~16777216,134217728~8388608,192937984~8388608,218103808~20971520,260046848~4194304,289406976~8388608,310378496~25165824,343932928~8388608,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.781 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~8388608,79691776~8388608,92274688~16777216,134217728~8388608,192937984~8388608,218103808~20971520,260046848~4194304,289406976~8388608,310378496~25165824,343932928~8388608,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.786 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~12582912,79691776~29360128,134217728~8388608,192937984~12582912,218103808~25165824,260046848~4194304,289406976~12582912,310378496~29360128,343932928~12582912,377487360~20971520,406847488~8388608] 2024-07-28T19:48:49.786 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 5 diff [0~12582912,33554432~12582912,71303168~16777216,92274688~16777216,134217728~8388608,155189248~8388608,192937984~8388608,218103808~20971520,260046848~12582912,289406976~8388608,306184192~29360128,339738624~12582912,377487360~16777216,402653184~8388608] 2024-07-28T19:48:49.787 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~12582912,71303168~16777216,92274688~16777216,134217728~8388608,155189248~8388608,192937984~8388608,218103808~20971520,260046848~12582912,289406976~8388608,306184192~29360128,339738624~12582912,377487360~16777216,402653184~8388608] 2024-07-28T19:48:49.792 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~16777216,71303168~37748736,134217728~8388608,155189248~8388608,192937984~12582912,218103808~25165824,260046848~16777216,289406976~12582912,306184192~50331648,377487360~20971520,402653184~12582912] 2024-07-28T19:48:49.792 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 6 diff [0~12582912,33554432~12582912,71303168~16777216,92274688~16777216,117440512~8388608,134217728~8388608,155189248~8388608,192937984~12582912,218103808~20971520,260046848~12582912,281018368~16777216,306184192~29360128,339738624~12582912,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.793 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~12582912,71303168~16777216,92274688~16777216,117440512~8388608,134217728~8388608,155189248~8388608,192937984~12582912,218103808~20971520,260046848~12582912,281018368~16777216,306184192~29360128,339738624~12582912,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.799 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~16777216,71303168~37748736,117440512~8388608,134217728~8388608,155189248~8388608,192937984~12582912,218103808~25165824,260046848~16777216,281018368~20971520,306184192~50331648,377487360~37748736] 2024-07-28T19:48:49.799 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 7 diff [0~16777216,33554432~12582912,71303168~37748736,117440512~8388608,134217728~16777216,155189248~8388608,180355072~4194304,192937984~12582912,218103808~20971520,260046848~12582912,281018368~16777216,306184192~29360128,339738624~12582912,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.799 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,71303168~37748736,117440512~8388608,134217728~16777216,155189248~8388608,180355072~4194304,192937984~12582912,218103808~20971520,260046848~12582912,281018368~16777216,306184192~29360128,339738624~12582912,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:49 smithi138 bash[67894]: cluster 2024-07-28T19:48:48.484891+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-07-28T19:48:49.808 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~16777216,71303168~37748736,117440512~8388608,134217728~16777216,155189248~8388608,180355072~8388608,192937984~12582912,218103808~25165824,260046848~16777216,281018368~20971520,306184192~50331648,369098752~46137344] 2024-07-28T19:48:49.808 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 8 diff [0~16777216,33554432~12582912,71303168~37748736,117440512~8388608,134217728~16777216,155189248~8388608,180355072~4194304,192937984~12582912,218103808~25165824,260046848~16777216,281018368~16777216,306184192~29360128,339738624~16777216,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.808 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,71303168~37748736,117440512~8388608,134217728~16777216,155189248~8388608,180355072~4194304,192937984~12582912,218103808~25165824,260046848~16777216,281018368~16777216,306184192~29360128,339738624~16777216,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.818 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~16777216,71303168~37748736,117440512~8388608,134217728~16777216,155189248~8388608,180355072~8388608,192937984~12582912,218103808~25165824,260046848~41943040,306184192~54525952,369098752~46137344] 2024-07-28T19:48:49.818 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 9 diff [0~16777216,33554432~12582912,50331648~8388608,71303168~41943040,117440512~12582912,134217728~16777216,155189248~8388608,167772160~8388608,180355072~8388608,192937984~12582912,209715200~37748736,260046848~16777216,281018368~16777216,306184192~29360128,339738624~16777216,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.818 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,50331648~8388608,71303168~41943040,117440512~12582912,134217728~16777216,155189248~8388608,167772160~8388608,180355072~8388608,192937984~12582912,209715200~37748736,260046848~16777216,281018368~16777216,306184192~29360128,339738624~16777216,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.829 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~25165824,71303168~79691776,155189248~8388608,167772160~8388608,180355072~8388608,192937984~12582912,209715200~41943040,260046848~41943040,306184192~54525952,369098752~46137344] 2024-07-28T19:48:49.829 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 10 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~20971520,180355072~8388608,192937984~12582912,209715200~46137344,260046848~16777216,281018368~16777216,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.830 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~20971520,180355072~8388608,192937984~12582912,209715200~46137344,260046848~16777216,281018368~16777216,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.841 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~79691776,155189248~20971520,180355072~8388608,192937984~12582912,209715200~92274688,306184192~54525952,369098752~46137344] 2024-07-28T19:48:49.842 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 11 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.842 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:49.854 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~79691776,155189248~50331648,209715200~155189248,369098752~46137344] 2024-07-28T19:48:49.855 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 12 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,364904448~50331648] 2024-07-28T19:48:49.855 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,364904448~50331648] 2024-07-28T19:48:49.867 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~79691776,155189248~50331648,209715200~205520896] 2024-07-28T19:48:49.868 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 13 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,364904448~50331648] 2024-07-28T19:48:49.868 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~46137344,260046848~16777216,281018368~20971520,306184192~29360128,339738624~20971520,364904448~50331648] 2024-07-28T19:48:49.882 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~79691776,155189248~260046848] 2024-07-28T19:48:49.882 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 14 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~46137344,260046848~16777216,281018368~20971520,306184192~54525952,364904448~50331648] 2024-07-28T19:48:49.882 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~46137344,260046848~16777216,281018368~20971520,306184192~54525952,364904448~50331648] 2024-07-28T19:48:49.899 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~79691776,155189248~260046848] 2024-07-28T19:48:49.899 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 15 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.899 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.915 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:49.915 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 16 diff [0~25165824,29360128~29360128,67108864~37748736,109051904~4194304,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.915 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~37748736,109051904~4194304,117440512~12582912,134217728~16777216,155189248~33554432,192937984~12582912,209715200~67108864,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.932 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:49.932 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 17 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~83886080,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.932 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~83886080,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.950 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:49.950 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 18 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.950 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.968 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:49.969 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to 19 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.969 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:49.989 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:49.989 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 3 diff [0~8388608,37748736~4194304,92274688~8388608,104857600~4194304,222298112~8388608,310378496~8388608,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.989 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,37748736~4194304,92274688~8388608,104857600~4194304,222298112~8388608,310378496~8388608,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.993 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,37748736~8388608,92274688~8388608,104857600~4194304,222298112~8388608,310378496~12582912,377487360~20971520,406847488~8388608] 2024-07-28T19:48:49.993 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 4 diff [0~12582912,33554432~8388608,92274688~16777216,218103808~20971520,310378496~8388608,327155712~4194304,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.993 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~8388608,92274688~16777216,218103808~20971520,310378496~8388608,327155712~4194304,377487360~16777216,406847488~8388608] 2024-07-28T19:48:49.997 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~12582912,92274688~16777216,218103808~25165824,310378496~12582912,327155712~8388608,377487360~20971520,406847488~8388608] 2024-07-28T19:48:49.997 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 5 diff [0~12582912,33554432~12582912,71303168~8388608,92274688~16777216,155189248~8388608,218103808~20971520,260046848~12582912,306184192~25165824,339738624~4194304,377487360~16777216,402653184~8388608] 2024-07-28T19:48:49.997 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~12582912,71303168~8388608,92274688~16777216,155189248~8388608,218103808~20971520,260046848~12582912,306184192~25165824,339738624~4194304,377487360~16777216,402653184~8388608] 2024-07-28T19:48:50.002 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~16777216,71303168~8388608,92274688~16777216,155189248~8388608,218103808~25165824,260046848~16777216,306184192~29360128,339738624~4194304,377487360~20971520,402653184~12582912] 2024-07-28T19:48:50.002 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 6 diff [0~12582912,33554432~12582912,71303168~12582912,92274688~16777216,117440512~8388608,155189248~8388608,192937984~4194304,218103808~20971520,260046848~12582912,281018368~8388608,306184192~25165824,339738624~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.002 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~12582912,71303168~12582912,92274688~16777216,117440512~8388608,155189248~8388608,192937984~4194304,218103808~20971520,260046848~12582912,281018368~8388608,306184192~25165824,339738624~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.008 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~16777216,71303168~12582912,92274688~16777216,117440512~8388608,155189248~8388608,192937984~4194304,218103808~25165824,260046848~16777216,281018368~8388608,306184192~29360128,339738624~4194304,377487360~37748736] 2024-07-28T19:48:50.008 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 7 diff [0~16777216,33554432~12582912,71303168~37748736,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~20971520,260046848~12582912,281018368~8388608,306184192~25165824,339738624~4194304,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.008 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,71303168~37748736,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~20971520,260046848~12582912,281018368~8388608,306184192~25165824,339738624~4194304,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.015 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~16777216,71303168~37748736,117440512~8388608,138412032~12582912,155189248~8388608,180355072~8388608,192937984~4194304,218103808~25165824,260046848~16777216,281018368~8388608,306184192~29360128,339738624~4194304,369098752~46137344] 2024-07-28T19:48:50.015 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 8 diff [0~16777216,33554432~12582912,71303168~37748736,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~25165824,260046848~16777216,281018368~8388608,306184192~25165824,339738624~4194304,352321536~8388608,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.016 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,71303168~37748736,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~25165824,260046848~16777216,281018368~8388608,306184192~25165824,339738624~4194304,352321536~8388608,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.025 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~16777216,71303168~37748736,117440512~8388608,138412032~12582912,155189248~8388608,180355072~8388608,192937984~4194304,218103808~25165824,260046848~29360128,306184192~29360128,339738624~4194304,352321536~8388608,369098752~46137344] 2024-07-28T19:48:50.025 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 9 diff [0~16777216,33554432~12582912,50331648~8388608,71303168~41943040,117440512~12582912,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~37748736,260046848~16777216,281018368~8388608,306184192~25165824,339738624~4194304,352321536~8388608,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.025 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,50331648~8388608,71303168~41943040,117440512~12582912,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~37748736,260046848~16777216,281018368~8388608,306184192~25165824,339738624~4194304,352321536~8388608,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.034 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~25165824,71303168~62914560,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~41943040,260046848~29360128,306184192~29360128,339738624~8388608,352321536~8388608,369098752~46137344] 2024-07-28T19:48:50.034 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 10 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~46137344,260046848~16777216,281018368~8388608,306184192~25165824,339738624~8388608,352321536~8388608,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.034 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~46137344,260046848~16777216,281018368~8388608,306184192~25165824,339738624~8388608,352321536~8388608,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.044 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~62914560,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~79691776,306184192~29360128,339738624~8388608,352321536~8388608,369098752~46137344] 2024-07-28T19:48:50.044 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 11 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.044 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,369098752~4194304,377487360~16777216,398458880~16777216] 2024-07-28T19:48:50.055 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~62914560,138412032~12582912,155189248~41943040,209715200~79691776,293601280~41943040,339738624~25165824,369098752~46137344] 2024-07-28T19:48:50.055 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 12 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~50331648] 2024-07-28T19:48:50.055 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~50331648] 2024-07-28T19:48:50.067 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~62914560,138412032~12582912,155189248~41943040,209715200~79691776,293601280~41943040,339738624~75497472] 2024-07-28T19:48:50.067 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 13 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~50331648] 2024-07-28T19:48:50.067 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~50331648] 2024-07-28T19:48:50.081 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~62914560,138412032~12582912,155189248~41943040,201326592~134217728,339738624~75497472] 2024-07-28T19:48:50.081 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 14 diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~50331648] 2024-07-28T19:48:50.081 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~20971520,33554432~25165824,71303168~41943040,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~50331648] 2024-07-28T19:48:50.096 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~62914560,138412032~12582912,155189248~41943040,201326592~213909504] 2024-07-28T19:48:50.096 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 15 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.096 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.113 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~67108864,138412032~12582912,155189248~41943040,201326592~213909504] 2024-07-28T19:48:50.113 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 16 diff [0~25165824,29360128~29360128,67108864~37748736,109051904~4194304,117440512~12582912,138412032~12582912,155189248~33554432,192937984~12582912,209715200~67108864,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.113 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~37748736,109051904~4194304,117440512~12582912,138412032~12582912,155189248~33554432,192937984~12582912,209715200~67108864,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.129 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~67108864,138412032~12582912,155189248~260046848] 2024-07-28T19:48:50.129 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 17 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~83886080,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.129 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~83886080,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.147 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:50.147 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 18 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.147 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.166 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:50.166 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to 19 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.166 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.185 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:50.186 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 4 diff [8388608~4194304,33554432~8388608,96468992~8388608,218103808~8388608,230686720~8388608,327155712~4194304,377487360~4194304] 2024-07-28T19:48:50.186 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~4194304,33554432~8388608,96468992~8388608,218103808~8388608,230686720~8388608,327155712~4194304,377487360~4194304] 2024-07-28T19:48:50.188 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~8388608,33554432~12582912,96468992~8388608,218103808~8388608,230686720~12582912,327155712~8388608,377487360~4194304] 2024-07-28T19:48:50.189 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 5 diff [8388608~4194304,33554432~12582912,71303168~8388608,96468992~8388608,155189248~8388608,218103808~8388608,230686720~8388608,260046848~12582912,306184192~4194304,318767104~12582912,339738624~4194304,377487360~8388608,402653184~4194304] 2024-07-28T19:48:50.189 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~4194304,33554432~12582912,71303168~8388608,96468992~8388608,155189248~8388608,218103808~8388608,230686720~8388608,260046848~12582912,306184192~4194304,318767104~12582912,339738624~4194304,377487360~8388608,402653184~4194304] 2024-07-28T19:48:50.192 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~8388608,33554432~16777216,71303168~8388608,96468992~8388608,155189248~8388608,218103808~8388608,230686720~12582912,260046848~16777216,306184192~8388608,318767104~16777216,339738624~4194304,377487360~8388608,402653184~8388608] 2024-07-28T19:48:50.192 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 6 diff [0~12582912,33554432~12582912,71303168~12582912,96468992~12582912,117440512~8388608,155189248~8388608,192937984~4194304,218103808~8388608,230686720~8388608,260046848~12582912,281018368~8388608,306184192~4194304,318767104~12582912,339738624~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.193 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,33554432~12582912,71303168~12582912,96468992~12582912,117440512~8388608,155189248~8388608,192937984~4194304,218103808~8388608,230686720~8388608,260046848~12582912,281018368~8388608,306184192~4194304,318767104~12582912,339738624~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.198 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,33554432~16777216,71303168~12582912,96468992~12582912,117440512~8388608,155189248~8388608,192937984~4194304,218103808~8388608,230686720~12582912,260046848~16777216,281018368~8388608,306184192~8388608,318767104~16777216,339738624~4194304,377487360~8388608,390070272~20971520] 2024-07-28T19:48:50.198 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 7 diff [0~16777216,33554432~12582912,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~8388608,230686720~8388608,260046848~12582912,281018368~8388608,306184192~4194304,318767104~12582912,339738624~4194304,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.198 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~8388608,230686720~8388608,260046848~12582912,281018368~8388608,306184192~4194304,318767104~12582912,339738624~4194304,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.205 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~16777216,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~8388608,192937984~4194304,218103808~25165824,260046848~16777216,281018368~8388608,306184192~8388608,318767104~16777216,339738624~4194304,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.206 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 8 diff [0~16777216,33554432~12582912,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~8388608,230686720~12582912,260046848~16777216,281018368~8388608,306184192~4194304,314572800~16777216,339738624~4194304,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.206 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~8388608,230686720~12582912,260046848~16777216,281018368~8388608,306184192~4194304,314572800~16777216,339738624~4194304,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.213 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~16777216,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~8388608,192937984~4194304,218103808~25165824,260046848~29360128,306184192~29360128,339738624~4194304,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.213 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 9 diff [0~16777216,33554432~12582912,50331648~8388608,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~16777216,281018368~8388608,306184192~4194304,314572800~16777216,339738624~4194304,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.213 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~12582912,50331648~8388608,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~16777216,281018368~8388608,306184192~4194304,314572800~16777216,339738624~4194304,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.222 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~25165824,71303168~20971520,96468992~37748736,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~41943040,260046848~29360128,306184192~29360128,339738624~8388608,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.222 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 10 diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,306184192~4194304,314572800~16777216,339738624~8388608,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.222 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,306184192~4194304,314572800~16777216,339738624~8388608,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.232 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~20971520,96468992~37748736,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~79691776,306184192~29360128,339738624~8388608,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.233 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 11 diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.233 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.244 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~20971520,96468992~37748736,138412032~12582912,155189248~41943040,209715200~79691776,293601280~41943040,339738624~25165824,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.245 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 12 diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~20971520,390070272~20971520] 2024-07-28T19:48:50.245 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~20971520,390070272~20971520] 2024-07-28T19:48:50.257 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~20971520,96468992~37748736,138412032~12582912,155189248~41943040,209715200~79691776,293601280~41943040,339738624~46137344,390070272~20971520] 2024-07-28T19:48:50.257 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 13 diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~46137344] 2024-07-28T19:48:50.257 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~46137344] 2024-07-28T19:48:50.270 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~20971520,96468992~37748736,138412032~12582912,155189248~41943040,201326592~134217728,339738624~71303168] 2024-07-28T19:48:50.271 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 14 diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~46137344] 2024-07-28T19:48:50.271 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,33554432~25165824,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~46137344] 2024-07-28T19:48:50.283 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,33554432~29360128,71303168~20971520,96468992~37748736,138412032~12582912,155189248~41943040,201326592~209715200] 2024-07-28T19:48:50.283 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 15 diff [0~25165824,29360128~29360128,67108864~25165824,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.283 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~25165824,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.298 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~25165824,96468992~37748736,138412032~12582912,155189248~41943040,201326592~209715200] 2024-07-28T19:48:50.298 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 16 diff [0~25165824,29360128~29360128,67108864~37748736,109051904~4194304,117440512~12582912,138412032~12582912,155189248~33554432,192937984~12582912,209715200~67108864,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.298 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~37748736,109051904~4194304,117440512~12582912,138412032~12582912,155189248~33554432,192937984~12582912,209715200~67108864,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.314 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~67108864,138412032~12582912,155189248~255852544] 2024-07-28T19:48:50.314 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 17 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~83886080,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.314 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~83886080,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.332 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~255852544] 2024-07-28T19:48:50.332 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 18 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.332 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.350 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:50.350 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to 19 diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.350 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.368 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:50.369 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 5 diff [41943040~8388608,71303168~8388608,155189248~8388608,260046848~12582912,306184192~4194304,318767104~8388608,339738624~4194304,381681664~4194304,402653184~4194304] 2024-07-28T19:48:50.369 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [41943040~8388608,71303168~8388608,155189248~8388608,260046848~12582912,306184192~4194304,318767104~8388608,339738624~4194304,381681664~4194304,402653184~4194304] 2024-07-28T19:48:50.371 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [41943040~8388608,71303168~8388608,155189248~8388608,260046848~16777216,306184192~8388608,318767104~8388608,339738624~4194304,381681664~4194304,402653184~8388608] 2024-07-28T19:48:50.372 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 6 diff [0~8388608,41943040~8388608,71303168~12582912,100663296~8388608,117440512~8388608,155189248~8388608,192937984~4194304,230686720~4194304,260046848~12582912,281018368~8388608,306184192~4194304,318767104~8388608,339738624~4194304,381681664~4194304,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.372 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,41943040~8388608,71303168~12582912,100663296~8388608,117440512~8388608,155189248~8388608,192937984~4194304,230686720~4194304,260046848~12582912,281018368~8388608,306184192~4194304,318767104~8388608,339738624~4194304,381681664~4194304,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.376 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,41943040~8388608,71303168~12582912,100663296~8388608,117440512~8388608,155189248~8388608,192937984~4194304,230686720~8388608,260046848~16777216,281018368~8388608,306184192~8388608,318767104~8388608,339738624~4194304,381681664~4194304,390070272~20971520] 2024-07-28T19:48:50.376 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 7 diff [0~8388608,12582912~8388608,41943040~8388608,71303168~20971520,100663296~8388608,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~8388608,230686720~4194304,260046848~12582912,281018368~8388608,306184192~4194304,318767104~8388608,339738624~4194304,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.377 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,12582912~8388608,41943040~8388608,71303168~20971520,100663296~8388608,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~8388608,230686720~4194304,260046848~12582912,281018368~8388608,306184192~4194304,318767104~8388608,339738624~4194304,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.382 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,12582912~8388608,41943040~8388608,71303168~20971520,100663296~8388608,117440512~8388608,138412032~12582912,155189248~8388608,180355072~8388608,192937984~4194304,218103808~20971520,260046848~16777216,281018368~8388608,306184192~8388608,318767104~8388608,339738624~4194304,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.382 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 8 diff [0~16777216,41943040~8388608,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~8388608,230686720~8388608,260046848~16777216,281018368~8388608,306184192~4194304,314572800~12582912,339738624~4194304,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.382 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,41943040~8388608,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~4194304,192937984~4194304,218103808~8388608,230686720~8388608,260046848~16777216,281018368~8388608,306184192~4194304,314572800~12582912,339738624~4194304,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.389 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,41943040~8388608,71303168~20971520,96468992~12582912,117440512~8388608,138412032~12582912,155189248~8388608,180355072~8388608,192937984~4194304,218103808~25165824,260046848~29360128,306184192~20971520,339738624~4194304,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.389 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 9 diff [0~16777216,41943040~16777216,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~16777216,281018368~8388608,306184192~4194304,314572800~12582912,339738624~4194304,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.389 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,41943040~16777216,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~16777216,281018368~8388608,306184192~4194304,314572800~12582912,339738624~4194304,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.397 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,41943040~16777216,71303168~20971520,96468992~37748736,138412032~12582912,155189248~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~41943040,260046848~29360128,306184192~20971520,339738624~8388608,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.397 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 10 diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,306184192~4194304,314572800~12582912,339738624~8388608,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.397 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,306184192~4194304,314572800~12582912,339738624~8388608,352321536~8388608,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.405 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,41943040~20971520,71303168~20971520,96468992~37748736,138412032~12582912,155189248~20971520,180355072~8388608,192937984~4194304,209715200~79691776,306184192~20971520,339738624~8388608,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.405 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 11 diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.405 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,369098752~4194304,377487360~8388608,390070272~4194304,398458880~12582912] 2024-07-28T19:48:50.415 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,41943040~20971520,71303168~20971520,96468992~37748736,138412032~12582912,155189248~41943040,209715200~79691776,293601280~41943040,339738624~25165824,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.415 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 12 diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~20971520,390070272~20971520] 2024-07-28T19:48:50.415 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~20971520,390070272~20971520] 2024-07-28T19:48:50.426 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,41943040~20971520,71303168~20971520,96468992~37748736,138412032~12582912,155189248~41943040,209715200~79691776,293601280~41943040,339738624~46137344,390070272~20971520] 2024-07-28T19:48:50.426 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 13 diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~46137344] 2024-07-28T19:48:50.427 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~46137344] 2024-07-28T19:48:50.439 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,41943040~20971520,71303168~20971520,96468992~37748736,138412032~12582912,155189248~41943040,201326592~134217728,339738624~71303168] 2024-07-28T19:48:50.439 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 14 diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~46137344] 2024-07-28T19:48:50.439 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,41943040~20971520,71303168~20971520,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~46137344] 2024-07-28T19:48:50.452 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,41943040~20971520,71303168~20971520,96468992~37748736,138412032~12582912,155189248~41943040,201326592~209715200] 2024-07-28T19:48:50.452 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 15 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.453 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~16777216,117440512~12582912,138412032~12582912,155189248~33554432,192937984~4194304,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.467 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~25165824,96468992~37748736,138412032~12582912,155189248~41943040,201326592~209715200] 2024-07-28T19:48:50.467 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 16 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,117440512~12582912,138412032~12582912,155189248~33554432,192937984~12582912,209715200~67108864,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.467 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,117440512~12582912,138412032~12582912,155189248~33554432,192937984~12582912,209715200~67108864,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.482 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~67108864,138412032~12582912,155189248~255852544] 2024-07-28T19:48:50.482 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 17 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~83886080,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.482 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~33554432,192937984~83886080,281018368~8388608,293601280~67108864,364904448~46137344] 2024-07-28T19:48:50.498 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~255852544] 2024-07-28T19:48:50.499 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 18 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.499 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.513 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:50.514 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to 19 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.514 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:50.530 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:50.530 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 6 diff [0~8388608,79691776~4194304,100663296~8388608,117440512~8388608,192937984~4194304,230686720~4194304,281018368~8388608,318767104~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.531 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,79691776~4194304,100663296~8388608,117440512~8388608,192937984~4194304,230686720~4194304,281018368~8388608,318767104~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.533 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,79691776~4194304,100663296~8388608,117440512~8388608,192937984~4194304,230686720~8388608,281018368~8388608,318767104~4194304,390070272~16777216] 2024-07-28T19:48:50.533 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 7 diff [0~8388608,12582912~8388608,79691776~12582912,100663296~8388608,117440512~8388608,138412032~12582912,180355072~4194304,192937984~4194304,218103808~8388608,230686720~4194304,281018368~8388608,318767104~4194304,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.533 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,12582912~8388608,79691776~12582912,100663296~8388608,117440512~8388608,138412032~12582912,180355072~4194304,192937984~4194304,218103808~8388608,230686720~4194304,281018368~8388608,318767104~4194304,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.537 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~8388608,12582912~8388608,79691776~12582912,100663296~8388608,117440512~8388608,138412032~12582912,180355072~8388608,192937984~4194304,218103808~20971520,281018368~8388608,318767104~4194304,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.537 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 8 diff [0~16777216,79691776~12582912,96468992~12582912,117440512~8388608,138412032~12582912,159383552~4194304,180355072~4194304,192937984~4194304,218103808~8388608,230686720~8388608,268435456~8388608,281018368~8388608,314572800~8388608,352321536~8388608,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.537 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,79691776~12582912,96468992~12582912,117440512~8388608,138412032~12582912,159383552~4194304,180355072~4194304,192937984~4194304,218103808~8388608,230686720~8388608,268435456~8388608,281018368~8388608,314572800~8388608,352321536~8388608,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.543 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,79691776~12582912,96468992~12582912,117440512~8388608,138412032~12582912,159383552~4194304,180355072~8388608,192937984~4194304,218103808~25165824,268435456~20971520,314572800~12582912,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.543 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 9 diff [0~16777216,50331648~8388608,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~4194304,268435456~8388608,281018368~8388608,314572800~8388608,339738624~4194304,352321536~8388608,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.543 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,50331648~8388608,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~4194304,268435456~8388608,281018368~8388608,314572800~8388608,339738624~4194304,352321536~8388608,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.549 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,50331648~8388608,79691776~12582912,96468992~37748736,138412032~12582912,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,209715200~41943040,260046848~29360128,314572800~12582912,339738624~8388608,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.549 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 10 diff [0~16777216,46137344~16777216,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~16777216,180355072~8388608,192937984~4194304,209715200~16777216,230686720~25165824,260046848~4194304,268435456~8388608,281018368~8388608,306184192~4194304,314572800~8388608,339738624~8388608,352321536~8388608,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.550 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~16777216,180355072~8388608,192937984~4194304,209715200~16777216,230686720~25165824,260046848~4194304,268435456~8388608,281018368~8388608,306184192~4194304,314572800~8388608,339738624~8388608,352321536~8388608,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.557 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,79691776~12582912,96468992~37748736,138412032~12582912,159383552~16777216,180355072~8388608,192937984~4194304,209715200~79691776,306184192~4194304,314572800~12582912,339738624~8388608,352321536~8388608,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.557 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 11 diff [0~16777216,46137344~16777216,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.557 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,369098752~4194304,377487360~4194304,390070272~4194304,398458880~8388608] 2024-07-28T19:48:50.565 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,79691776~12582912,96468992~37748736,138412032~12582912,159383552~37748736,209715200~79691776,293601280~16777216,314572800~20971520,339738624~25165824,369098752~16777216,390070272~20971520] 2024-07-28T19:48:50.565 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 12 diff [0~16777216,46137344~16777216,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~16777216,390070272~16777216] 2024-07-28T19:48:50.565 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~16777216,390070272~16777216] 2024-07-28T19:48:50.574 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,79691776~12582912,96468992~37748736,138412032~12582912,159383552~37748736,209715200~79691776,293601280~16777216,314572800~20971520,339738624~46137344,390070272~20971520] 2024-07-28T19:48:50.574 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 13 diff [0~16777216,46137344~16777216,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,201326592~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.574 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,79691776~12582912,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,201326592~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.584 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,79691776~12582912,96468992~37748736,138412032~12582912,159383552~37748736,201326592~134217728,339738624~71303168] 2024-07-28T19:48:50.584 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 14 diff [0~16777216,46137344~16777216,75497472~16777216,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.584 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,75497472~16777216,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.595 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,75497472~16777216,96468992~37748736,138412032~12582912,159383552~37748736,201326592~209715200] 2024-07-28T19:48:50.595 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 15 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.595 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~16777216,117440512~12582912,138412032~12582912,159383552~29360128,192937984~4194304,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.607 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~25165824,96468992~37748736,138412032~12582912,159383552~37748736,201326592~209715200] 2024-07-28T19:48:50.608 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 16 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,117440512~12582912,138412032~12582912,159383552~29360128,192937984~12582912,209715200~67108864,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.608 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,117440512~12582912,138412032~12582912,159383552~29360128,192937984~12582912,209715200~67108864,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.620 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~67108864,138412032~12582912,159383552~251658240] 2024-07-28T19:48:50.620 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 17 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~29360128,192937984~83886080,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.620 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~29360128,192937984~83886080,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.634 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~251658240] 2024-07-28T19:48:50.634 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 18 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.634 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.648 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:50.648 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to 19 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.649 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.664 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:50.664 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 7 diff [12582912~8388608,83886080~8388608,138412032~12582912,180355072~4194304,218103808~8388608,369098752~4194304,377487360~4194304,402653184~4194304] 2024-07-28T19:48:50.664 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [12582912~8388608,83886080~8388608,138412032~12582912,180355072~4194304,218103808~8388608,369098752~4194304,377487360~4194304,402653184~4194304] 2024-07-28T19:48:50.668 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [12582912~8388608,83886080~8388608,138412032~12582912,180355072~8388608,218103808~12582912,369098752~16777216,402653184~8388608] 2024-07-28T19:48:50.668 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 8 diff [8388608~8388608,83886080~8388608,96468992~4194304,138412032~12582912,159383552~4194304,180355072~4194304,218103808~8388608,234881024~4194304,268435456~8388608,314572800~8388608,352321536~8388608,369098752~4194304,377487360~4194304,398458880~8388608] 2024-07-28T19:48:50.668 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~8388608,83886080~8388608,96468992~4194304,138412032~12582912,159383552~4194304,180355072~4194304,218103808~8388608,234881024~4194304,268435456~8388608,314572800~8388608,352321536~8388608,369098752~4194304,377487360~4194304,398458880~8388608] 2024-07-28T19:48:50.672 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~12582912,83886080~8388608,96468992~8388608,138412032~12582912,159383552~4194304,180355072~8388608,218103808~12582912,234881024~8388608,268435456~12582912,314572800~12582912,352321536~8388608,369098752~16777216,398458880~12582912] 2024-07-28T19:48:50.673 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 9 diff [8388608~8388608,50331648~8388608,83886080~8388608,96468992~4194304,109051904~4194304,121634816~8388608,138412032~12582912,159383552~4194304,167772160~8388608,180355072~8388608,209715200~16777216,230686720~16777216,260046848~4194304,268435456~8388608,314572800~8388608,339738624~4194304,352321536~8388608,369098752~4194304,377487360~4194304,398458880~8388608] 2024-07-28T19:48:50.673 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~8388608,50331648~8388608,83886080~8388608,96468992~4194304,109051904~4194304,121634816~8388608,138412032~12582912,159383552~4194304,167772160~8388608,180355072~8388608,209715200~16777216,230686720~16777216,260046848~4194304,268435456~8388608,314572800~8388608,339738624~4194304,352321536~8388608,369098752~4194304,377487360~4194304,398458880~8388608] 2024-07-28T19:48:50.678 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~12582912,50331648~8388608,83886080~8388608,96468992~8388608,109051904~8388608,121634816~12582912,138412032~12582912,159383552~4194304,167772160~8388608,180355072~8388608,209715200~41943040,260046848~20971520,314572800~12582912,339738624~8388608,352321536~8388608,369098752~16777216,398458880~12582912] 2024-07-28T19:48:50.678 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 10 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~4194304,109051904~4194304,121634816~8388608,138412032~12582912,159383552~16777216,180355072~8388608,209715200~16777216,230686720~25165824,260046848~4194304,268435456~8388608,306184192~4194304,314572800~8388608,339738624~8388608,352321536~8388608,369098752~4194304,377487360~4194304,398458880~8388608] 2024-07-28T19:48:50.678 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~4194304,109051904~4194304,121634816~8388608,138412032~12582912,159383552~16777216,180355072~8388608,209715200~16777216,230686720~25165824,260046848~4194304,268435456~8388608,306184192~4194304,314572800~8388608,339738624~8388608,352321536~8388608,369098752~4194304,377487360~4194304,398458880~8388608] 2024-07-28T19:48:50.684 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,83886080~8388608,96468992~8388608,109051904~8388608,121634816~12582912,138412032~12582912,159383552~16777216,180355072~8388608,209715200~71303168,306184192~4194304,314572800~12582912,339738624~8388608,352321536~8388608,369098752~16777216,398458880~12582912] 2024-07-28T19:48:50.685 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 11 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,209715200~16777216,230686720~25165824,260046848~16777216,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,369098752~4194304,377487360~4194304,398458880~8388608] 2024-07-28T19:48:50.685 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,209715200~16777216,230686720~25165824,260046848~16777216,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,369098752~4194304,377487360~4194304,398458880~8388608] 2024-07-28T19:48:50.692 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,138412032~12582912,159383552~33554432,209715200~71303168,293601280~16777216,314572800~20971520,339738624~25165824,369098752~16777216,398458880~12582912] 2024-07-28T19:48:50.692 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 12 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,209715200~16777216,230686720~25165824,260046848~16777216,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~16777216,394264576~12582912] 2024-07-28T19:48:50.692 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,209715200~16777216,230686720~25165824,260046848~16777216,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~16777216,394264576~12582912] 2024-07-28T19:48:50.700 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,138412032~12582912,159383552~33554432,209715200~71303168,293601280~16777216,314572800~20971520,339738624~46137344,394264576~16777216] 2024-07-28T19:48:50.700 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 13 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,201326592~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.701 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,201326592~4194304,209715200~16777216,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.709 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,138412032~12582912,159383552~33554432,201326592~134217728,339738624~71303168] 2024-07-28T19:48:50.709 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 14 diff [0~16777216,46137344~16777216,75497472~4194304,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.710 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,75497472~4194304,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,201326592~4194304,209715200~46137344,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.719 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,75497472~16777216,96468992~20971520,121634816~12582912,138412032~12582912,159383552~33554432,201326592~209715200] 2024-07-28T19:48:50.720 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 15 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.720 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~8388608,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,201326592~4194304,209715200~67108864,281018368~8388608,293601280~67108864,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.731 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~25165824,96468992~20971520,121634816~12582912,138412032~12582912,159383552~33554432,201326592~209715200] 2024-07-28T19:48:50.731 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 16 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,197132288~8388608,209715200~67108864,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.731 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,121634816~8388608,138412032~12582912,159383552~29360128,197132288~8388608,209715200~67108864,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.743 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~50331648,121634816~12582912,138412032~12582912,159383552~33554432,197132288~213909504] 2024-07-28T19:48:50.743 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 17 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,121634816~8388608,134217728~16777216,159383552~29360128,197132288~79691776,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.743 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,121634816~8388608,134217728~16777216,159383552~29360128,197132288~79691776,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:50 smithi100 bash[80199]: cluster 2024-07-28T19:48:49.490800+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-07-28T19:48:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:50 smithi100 bash[80199]: cluster 2024-07-28T19:48:50.064503+0000 mgr.y (mgr.25358) 487 : cluster [DBG] pgmap v1032: 105 pgs: 105 active+clean; 806 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 241 KiB/s rd, 34 MiB/s wr, 368 op/s 2024-07-28T19:48:50.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:50 smithi100 bash[82644]: cluster 2024-07-28T19:48:49.490800+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-07-28T19:48:50.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:50 smithi100 bash[82644]: cluster 2024-07-28T19:48:50.064503+0000 mgr.y (mgr.25358) 487 : cluster [DBG] pgmap v1032: 105 pgs: 105 active+clean; 806 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 241 KiB/s rd, 34 MiB/s wr, 368 op/s 2024-07-28T19:48:50.755 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~50331648,121634816~29360128,159383552~33554432,197132288~213909504] 2024-07-28T19:48:50.755 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 18 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.755 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.768 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:50.768 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to 19 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.769 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.783 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:50.783 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 8 diff [8388608~8388608,96468992~4194304,146800640~4194304,159383552~4194304,234881024~4194304,268435456~8388608,314572800~8388608,352321536~8388608,398458880~8388608] 2024-07-28T19:48:50.783 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~8388608,96468992~4194304,146800640~4194304,159383552~4194304,234881024~4194304,268435456~8388608,314572800~8388608,352321536~8388608,398458880~8388608] 2024-07-28T19:48:50.787 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~12582912,96468992~8388608,146800640~4194304,159383552~4194304,234881024~8388608,268435456~12582912,314572800~12582912,352321536~8388608,398458880~8388608] 2024-07-28T19:48:50.787 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 9 diff [8388608~8388608,50331648~8388608,96468992~4194304,109051904~4194304,121634816~8388608,146800640~4194304,159383552~4194304,167772160~8388608,180355072~4194304,209715200~8388608,230686720~16777216,260046848~4194304,268435456~8388608,314572800~8388608,339738624~4194304,352321536~8388608,398458880~8388608] 2024-07-28T19:48:50.787 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~8388608,50331648~8388608,96468992~4194304,109051904~4194304,121634816~8388608,146800640~4194304,159383552~4194304,167772160~8388608,180355072~4194304,209715200~8388608,230686720~16777216,260046848~4194304,268435456~8388608,314572800~8388608,339738624~4194304,352321536~8388608,398458880~8388608] 2024-07-28T19:48:50.792 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~12582912,50331648~8388608,96468992~8388608,109051904~8388608,121634816~12582912,146800640~4194304,159383552~4194304,167772160~8388608,180355072~8388608,209715200~12582912,230686720~20971520,260046848~20971520,314572800~12582912,339738624~8388608,352321536~8388608,398458880~8388608] 2024-07-28T19:48:50.793 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 10 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~4194304,109051904~4194304,121634816~8388608,146800640~4194304,159383552~16777216,180355072~4194304,209715200~8388608,230686720~25165824,260046848~4194304,268435456~8388608,306184192~4194304,314572800~8388608,339738624~8388608,352321536~8388608,398458880~8388608] 2024-07-28T19:48:50.793 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~4194304,109051904~4194304,121634816~8388608,146800640~4194304,159383552~16777216,180355072~4194304,209715200~8388608,230686720~25165824,260046848~4194304,268435456~8388608,306184192~4194304,314572800~8388608,339738624~8388608,352321536~8388608,398458880~8388608] 2024-07-28T19:48:50.797 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,83886080~8388608,96468992~8388608,109051904~8388608,121634816~12582912,146800640~4194304,159383552~16777216,180355072~8388608,209715200~12582912,230686720~50331648,306184192~4194304,314572800~12582912,339738624~8388608,352321536~8388608,398458880~12582912] 2024-07-28T19:48:50.797 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 11 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,209715200~8388608,230686720~25165824,260046848~16777216,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,398458880~8388608] 2024-07-28T19:48:50.798 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,209715200~8388608,230686720~25165824,260046848~16777216,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,398458880~8388608] 2024-07-28T19:48:50.804 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,146800640~4194304,159383552~33554432,209715200~12582912,230686720~50331648,293601280~16777216,314572800~20971520,339738624~25165824,398458880~12582912] 2024-07-28T19:48:50.804 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 12 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,209715200~8388608,230686720~25165824,260046848~16777216,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~16777216,394264576~12582912] 2024-07-28T19:48:50.804 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,209715200~8388608,230686720~25165824,260046848~16777216,293601280~8388608,306184192~4194304,314572800~16777216,339738624~20971520,364904448~16777216,394264576~12582912] 2024-07-28T19:48:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:50 smithi138 bash[67894]: cluster 2024-07-28T19:48:49.490800+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-07-28T19:48:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:50 smithi138 bash[67894]: cluster 2024-07-28T19:48:50.064503+0000 mgr.y (mgr.25358) 487 : cluster [DBG] pgmap v1032: 105 pgs: 105 active+clean; 806 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 241 KiB/s rd, 34 MiB/s wr, 368 op/s 2024-07-28T19:48:50.812 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,146800640~4194304,159383552~33554432,209715200~12582912,230686720~50331648,293601280~16777216,314572800~20971520,339738624~46137344,394264576~16777216] 2024-07-28T19:48:50.812 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 13 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.812 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,230686720~25165824,260046848~16777216,281018368~8388608,293601280~8388608,306184192~25165824,339738624~20971520,364904448~16777216,385875968~20971520] 2024-07-28T19:48:50.820 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,146800640~4194304,159383552~33554432,201326592~20971520,230686720~104857600,339738624~71303168] 2024-07-28T19:48:50.821 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 14 diff [0~16777216,46137344~16777216,75497472~4194304,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~29360128,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.821 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,75497472~4194304,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~29360128,260046848~16777216,281018368~8388608,293601280~8388608,306184192~54525952,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.830 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~20971520,46137344~16777216,75497472~16777216,96468992~20971520,121634816~12582912,146800640~4194304,159383552~33554432,201326592~20971520,226492416~184549376] 2024-07-28T19:48:50.831 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 15 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~50331648,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.831 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~50331648,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.841 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~25165824,96468992~20971520,121634816~12582912,146800640~4194304,159383552~33554432,201326592~20971520,226492416~184549376] 2024-07-28T19:48:50.841 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 16 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,197132288~8388608,209715200~8388608,226492416~50331648,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.841 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,197132288~8388608,209715200~8388608,226492416~50331648,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.852 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~50331648,121634816~12582912,146800640~4194304,159383552~33554432,197132288~25165824,226492416~184549376] 2024-07-28T19:48:50.852 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 17 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,121634816~8388608,134217728~4194304,146800640~4194304,159383552~29360128,197132288~20971520,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.852 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,121634816~8388608,134217728~4194304,146800640~4194304,159383552~29360128,197132288~20971520,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.864 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~50331648,121634816~20971520,146800640~4194304,159383552~33554432,197132288~213909504] 2024-07-28T19:48:50.864 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 18 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.864 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.877 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:50.877 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to 19 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.877 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:50.891 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:50.891 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 9 diff [50331648~8388608,109051904~4194304,121634816~8388608,167772160~8388608,180355072~4194304,209715200~8388608,230686720~8388608,243269632~4194304,260046848~4194304,339738624~4194304] 2024-07-28T19:48:50.891 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [50331648~8388608,109051904~4194304,121634816~8388608,167772160~8388608,180355072~4194304,209715200~8388608,230686720~8388608,243269632~4194304,260046848~4194304,339738624~4194304] 2024-07-28T19:48:50.895 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [50331648~8388608,109051904~8388608,121634816~12582912,167772160~8388608,180355072~8388608,209715200~12582912,230686720~8388608,243269632~8388608,260046848~8388608,339738624~8388608] 2024-07-28T19:48:50.895 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 10 diff [0~8388608,46137344~16777216,83886080~8388608,109051904~4194304,121634816~8388608,159383552~16777216,180355072~4194304,209715200~8388608,230686720~8388608,243269632~12582912,260046848~4194304,306184192~4194304,339738624~8388608,402653184~4194304] 2024-07-28T19:48:50.895 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,46137344~16777216,83886080~8388608,109051904~4194304,121634816~8388608,159383552~16777216,180355072~4194304,209715200~8388608,230686720~8388608,243269632~12582912,260046848~4194304,306184192~4194304,339738624~8388608,402653184~4194304] 2024-07-28T19:48:50.899 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~12582912,46137344~16777216,83886080~8388608,109051904~8388608,121634816~12582912,159383552~16777216,180355072~8388608,209715200~12582912,230686720~8388608,243269632~25165824,306184192~4194304,339738624~8388608,402653184~8388608] 2024-07-28T19:48:50.899 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 11 diff [0~8388608,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,159383552~29360128,209715200~8388608,230686720~8388608,243269632~12582912,260046848~12582912,293601280~8388608,306184192~4194304,322961408~8388608,339738624~20971520,402653184~4194304] 2024-07-28T19:48:50.900 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,159383552~29360128,209715200~8388608,230686720~8388608,243269632~12582912,260046848~12582912,293601280~8388608,306184192~4194304,322961408~8388608,339738624~20971520,402653184~4194304] 2024-07-28T19:48:50.906 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~12582912,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,159383552~33554432,209715200~12582912,230686720~8388608,243269632~33554432,293601280~16777216,322961408~12582912,339738624~25165824,402653184~8388608] 2024-07-28T19:48:50.906 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 12 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,159383552~29360128,209715200~8388608,230686720~25165824,260046848~12582912,293601280~8388608,306184192~4194304,322961408~8388608,339738624~20971520,364904448~16777216,394264576~4194304,402653184~4194304] 2024-07-28T19:48:50.906 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,159383552~29360128,209715200~8388608,230686720~25165824,260046848~12582912,293601280~8388608,306184192~4194304,322961408~8388608,339738624~20971520,364904448~16777216,394264576~4194304,402653184~4194304] 2024-07-28T19:48:50.912 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,159383552~33554432,209715200~12582912,230686720~46137344,293601280~16777216,322961408~12582912,339738624~46137344,394264576~16777216] 2024-07-28T19:48:50.912 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 13 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,230686720~25165824,260046848~12582912,281018368~8388608,293601280~8388608,306184192~12582912,322961408~8388608,339738624~20971520,364904448~16777216,385875968~12582912,402653184~4194304] 2024-07-28T19:48:50.912 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,230686720~25165824,260046848~12582912,281018368~8388608,293601280~8388608,306184192~12582912,322961408~8388608,339738624~20971520,364904448~16777216,385875968~12582912,402653184~4194304] 2024-07-28T19:48:50.919 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~16777216,83886080~8388608,96468992~20971520,121634816~12582912,146800640~4194304,159383552~33554432,201326592~20971520,230686720~46137344,281018368~54525952,339738624~71303168] 2024-07-28T19:48:50.919 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 14 diff [0~16777216,46137344~16777216,75497472~4194304,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~29360128,260046848~12582912,281018368~8388608,293601280~8388608,306184192~54525952,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.919 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,75497472~4194304,83886080~8388608,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~29360128,260046848~12582912,281018368~8388608,293601280~8388608,306184192~54525952,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.927 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~16777216,75497472~16777216,96468992~20971520,121634816~12582912,146800640~4194304,159383552~33554432,201326592~20971520,226492416~50331648,281018368~130023424] 2024-07-28T19:48:50.928 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 15 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~46137344,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.928 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~8388608,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~46137344,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.937 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~25165824,96468992~20971520,121634816~12582912,146800640~4194304,159383552~33554432,201326592~20971520,226492416~50331648,281018368~130023424] 2024-07-28T19:48:50.937 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 16 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,197132288~8388608,209715200~8388608,226492416~50331648,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.937 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,121634816~8388608,146800640~4194304,159383552~29360128,197132288~8388608,209715200~8388608,226492416~50331648,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.947 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~50331648,121634816~12582912,146800640~4194304,159383552~33554432,197132288~25165824,226492416~184549376] 2024-07-28T19:48:50.947 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 17 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,121634816~8388608,134217728~4194304,146800640~4194304,159383552~29360128,197132288~20971520,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.947 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,121634816~8388608,134217728~4194304,146800640~4194304,159383552~29360128,197132288~20971520,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.958 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~50331648,121634816~20971520,146800640~4194304,159383552~33554432,197132288~213909504] 2024-07-28T19:48:50.958 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 18 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.958 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.970 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:50.970 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to 19 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.970 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:50.983 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:50.983 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 10 diff [0~8388608,46137344~12582912,83886080~8388608,159383552~8388608,247463936~8388608,306184192~4194304,339738624~8388608,402653184~4194304] 2024-07-28T19:48:50.983 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,46137344~12582912,83886080~8388608,159383552~8388608,247463936~8388608,306184192~4194304,339738624~8388608,402653184~4194304] 2024-07-28T19:48:50.986 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~12582912,46137344~16777216,83886080~8388608,159383552~8388608,247463936~12582912,306184192~4194304,339738624~8388608,402653184~8388608] 2024-07-28T19:48:50.986 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 11 diff [0~8388608,46137344~16777216,83886080~8388608,96468992~8388608,159383552~8388608,176160768~12582912,247463936~8388608,264241152~8388608,293601280~8388608,306184192~4194304,322961408~8388608,339738624~12582912,356515840~4194304,402653184~4194304] 2024-07-28T19:48:50.986 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~8388608,46137344~16777216,83886080~8388608,96468992~8388608,159383552~8388608,176160768~12582912,247463936~8388608,264241152~8388608,293601280~8388608,306184192~4194304,322961408~8388608,339738624~12582912,356515840~4194304,402653184~4194304] 2024-07-28T19:48:50.991 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~12582912,46137344~16777216,83886080~8388608,96468992~12582912,159383552~8388608,176160768~16777216,247463936~12582912,264241152~12582912,293601280~16777216,322961408~12582912,339738624~25165824,402653184~8388608] 2024-07-28T19:48:50.991 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 12 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,113246208~4194304,159383552~8388608,176160768~12582912,209715200~8388608,239075328~4194304,247463936~8388608,264241152~8388608,293601280~8388608,306184192~4194304,322961408~8388608,339738624~12582912,356515840~4194304,364904448~16777216,394264576~4194304,402653184~4194304] 2024-07-28T19:48:50.991 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,113246208~4194304,159383552~8388608,176160768~12582912,209715200~8388608,239075328~4194304,247463936~8388608,264241152~8388608,293601280~8388608,306184192~4194304,322961408~8388608,339738624~12582912,356515840~4194304,364904448~16777216,394264576~4194304,402653184~4194304] 2024-07-28T19:48:50.996 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~16777216,83886080~8388608,96468992~12582912,113246208~4194304,159383552~8388608,176160768~16777216,209715200~12582912,239075328~4194304,247463936~12582912,264241152~12582912,293601280~16777216,322961408~12582912,339738624~46137344,394264576~16777216] 2024-07-28T19:48:50.996 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 13 diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,113246208~4194304,146800640~4194304,159383552~8388608,176160768~12582912,201326592~4194304,209715200~8388608,239075328~4194304,247463936~8388608,264241152~8388608,281018368~8388608,293601280~8388608,306184192~12582912,322961408~8388608,339738624~12582912,356515840~4194304,364904448~16777216,385875968~12582912,402653184~4194304] 2024-07-28T19:48:50.996 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,83886080~8388608,96468992~8388608,113246208~4194304,146800640~4194304,159383552~8388608,176160768~12582912,201326592~4194304,209715200~8388608,239075328~4194304,247463936~8388608,264241152~8388608,281018368~8388608,293601280~8388608,306184192~12582912,322961408~8388608,339738624~12582912,356515840~4194304,364904448~16777216,385875968~12582912,402653184~4194304] 2024-07-28T19:48:51.002 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~16777216,83886080~8388608,96468992~12582912,113246208~4194304,146800640~4194304,159383552~8388608,176160768~16777216,201326592~20971520,239075328~4194304,247463936~12582912,264241152~12582912,281018368~54525952,339738624~71303168] 2024-07-28T19:48:51.002 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 14 diff [0~16777216,46137344~16777216,75497472~4194304,83886080~8388608,96468992~8388608,113246208~4194304,146800640~4194304,159383552~8388608,176160768~12582912,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~8388608,264241152~8388608,281018368~8388608,293601280~8388608,306184192~46137344,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.003 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~16777216,46137344~16777216,75497472~4194304,83886080~8388608,96468992~8388608,113246208~4194304,146800640~4194304,159383552~8388608,176160768~12582912,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~8388608,264241152~8388608,281018368~8388608,293601280~8388608,306184192~46137344,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.010 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~16777216,75497472~16777216,96468992~12582912,113246208~4194304,146800640~4194304,159383552~8388608,176160768~16777216,201326592~20971520,226492416~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~130023424] 2024-07-28T19:48:51.010 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 15 diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~8388608,113246208~4194304,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,264241152~8388608,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.010 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~25165824,96468992~8388608,113246208~4194304,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,264241152~8388608,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.018 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~25165824,96468992~12582912,113246208~4194304,146800640~4194304,159383552~33554432,201326592~20971520,226492416~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~130023424] 2024-07-28T19:48:51.018 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 16 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,146800640~4194304,159383552~29360128,197132288~8388608,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.019 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~37748736,109051904~4194304,146800640~4194304,159383552~29360128,197132288~8388608,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.027 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~50331648,146800640~4194304,159383552~33554432,197132288~25165824,226492416~8388608,239075328~4194304,247463936~12582912,264241152~146800640] 2024-07-28T19:48:51.028 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 17 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,134217728~4194304,146800640~4194304,159383552~29360128,197132288~20971520,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.028 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,134217728~4194304,146800640~4194304,159383552~29360128,197132288~20971520,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.037 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~50331648,134217728~8388608,146800640~4194304,159383552~33554432,197132288~37748736,239075328~4194304,247463936~12582912,264241152~146800640] 2024-07-28T19:48:51.037 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 18 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.038 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.048 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:51.048 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to 19 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.049 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.061 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:51.061 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 11 diff [46137344~8388608,96468992~8388608,176160768~12582912,264241152~8388608,293601280~8388608,322961408~8388608,343932928~16777216] 2024-07-28T19:48:51.061 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [46137344~8388608,96468992~8388608,176160768~12582912,264241152~8388608,293601280~8388608,322961408~8388608,343932928~16777216] 2024-07-28T19:48:51.065 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [46137344~12582912,96468992~12582912,176160768~16777216,264241152~12582912,293601280~12582912,322961408~12582912,343932928~20971520] 2024-07-28T19:48:51.065 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 12 diff [0~4194304,8388608~8388608,46137344~8388608,96468992~8388608,113246208~4194304,176160768~12582912,209715200~8388608,239075328~4194304,264241152~8388608,293601280~8388608,322961408~8388608,343932928~16777216,364904448~16777216,394264576~4194304] 2024-07-28T19:48:51.066 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,8388608~8388608,46137344~8388608,96468992~8388608,113246208~4194304,176160768~12582912,209715200~8388608,239075328~4194304,264241152~8388608,293601280~8388608,322961408~8388608,343932928~16777216,364904448~16777216,394264576~4194304] 2024-07-28T19:48:51.070 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~12582912,96468992~12582912,113246208~4194304,176160768~16777216,209715200~12582912,239075328~4194304,264241152~12582912,293601280~12582912,322961408~12582912,343932928~41943040,394264576~8388608] 2024-07-28T19:48:51.070 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 13 diff [0~4194304,8388608~8388608,46137344~12582912,96468992~8388608,113246208~4194304,146800640~4194304,159383552~4194304,176160768~12582912,201326592~4194304,209715200~8388608,239075328~4194304,247463936~4194304,264241152~8388608,281018368~8388608,293601280~8388608,310378496~8388608,322961408~8388608,343932928~16777216,364904448~16777216,385875968~12582912] 2024-07-28T19:48:51.070 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,8388608~8388608,46137344~12582912,96468992~8388608,113246208~4194304,146800640~4194304,159383552~4194304,176160768~12582912,201326592~4194304,209715200~8388608,239075328~4194304,247463936~4194304,264241152~8388608,281018368~8388608,293601280~8388608,310378496~8388608,322961408~8388608,343932928~16777216,364904448~16777216,385875968~12582912] 2024-07-28T19:48:51.076 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~12582912,96468992~12582912,113246208~4194304,146800640~4194304,159383552~8388608,176160768~16777216,201326592~20971520,239075328~4194304,247463936~8388608,264241152~12582912,281018368~25165824,310378496~25165824,343932928~58720256] 2024-07-28T19:48:51.076 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 14 diff [0~4194304,8388608~8388608,46137344~12582912,75497472~4194304,96468992~8388608,113246208~4194304,146800640~4194304,159383552~4194304,176160768~12582912,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~4194304,264241152~8388608,281018368~8388608,293601280~8388608,310378496~29360128,343932928~16777216,364904448~16777216,385875968~16777216,406847488~4194304] 2024-07-28T19:48:51.076 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,8388608~8388608,46137344~12582912,75497472~4194304,96468992~8388608,113246208~4194304,146800640~4194304,159383552~4194304,176160768~12582912,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~4194304,264241152~8388608,281018368~8388608,293601280~8388608,310378496~29360128,343932928~16777216,364904448~16777216,385875968~16777216,406847488~4194304] 2024-07-28T19:48:51.083 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,46137344~12582912,75497472~8388608,96468992~12582912,113246208~4194304,146800640~4194304,159383552~8388608,176160768~16777216,201326592~20971520,226492416~8388608,239075328~4194304,247463936~8388608,264241152~12582912,281018368~25165824,310378496~100663296] 2024-07-28T19:48:51.083 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 15 diff [0~4194304,8388608~16777216,29360128~4194304,41943040~16777216,67108864~16777216,96468992~8388608,113246208~4194304,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,264241152~8388608,281018368~8388608,293601280~12582912,310378496~29360128,343932928~16777216,364904448~16777216,385875968~16777216,406847488~4194304] 2024-07-28T19:48:51.083 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,8388608~16777216,29360128~4194304,41943040~16777216,67108864~16777216,96468992~8388608,113246208~4194304,146800640~4194304,159383552~29360128,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,264241152~8388608,281018368~8388608,293601280~12582912,310378496~29360128,343932928~16777216,364904448~16777216,385875968~16777216,406847488~4194304] 2024-07-28T19:48:51.090 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~16777216,96468992~12582912,113246208~4194304,146800640~4194304,159383552~33554432,201326592~20971520,226492416~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~130023424] 2024-07-28T19:48:51.091 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 16 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~16777216,88080384~16777216,109051904~4194304,146800640~4194304,159383552~29360128,197132288~8388608,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~12582912,310378496~29360128,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.091 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~16777216,88080384~16777216,109051904~4194304,146800640~4194304,159383552~29360128,197132288~8388608,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~12582912,310378496~29360128,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.100 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~16777216,88080384~29360128,146800640~4194304,159383552~33554432,197132288~25165824,226492416~8388608,239075328~4194304,247463936~12582912,264241152~146800640] 2024-07-28T19:48:51.100 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 17 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~16777216,88080384~25165824,134217728~4194304,146800640~4194304,159383552~29360128,197132288~20971520,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~12582912,310378496~29360128,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.100 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~16777216,88080384~25165824,134217728~4194304,146800640~4194304,159383552~29360128,197132288~20971520,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~12582912,310378496~29360128,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.109 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~16777216,88080384~29360128,134217728~8388608,146800640~4194304,159383552~33554432,197132288~37748736,239075328~4194304,247463936~12582912,264241152~146800640] 2024-07-28T19:48:51.109 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 18 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~16777216,88080384~25165824,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.109 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~16777216,88080384~25165824,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.119 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~16777216,88080384~41943040,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:51.119 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to 19 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.119 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.131 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:51.131 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 12 diff [0~4194304,8388608~8388608,113246208~4194304,209715200~8388608,239075328~4194304,293601280~8388608,364904448~16777216,394264576~4194304] 2024-07-28T19:48:51.131 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,8388608~8388608,113246208~4194304,209715200~8388608,239075328~4194304,293601280~8388608,364904448~16777216,394264576~4194304] 2024-07-28T19:48:51.134 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,113246208~4194304,209715200~12582912,239075328~4194304,293601280~12582912,364904448~20971520,394264576~8388608] 2024-07-28T19:48:51.134 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 13 diff [0~4194304,8388608~8388608,54525952~4194304,113246208~4194304,146800640~4194304,159383552~4194304,201326592~4194304,209715200~8388608,239075328~4194304,247463936~4194304,281018368~8388608,293601280~8388608,310378496~8388608,364904448~16777216,385875968~12582912] 2024-07-28T19:48:51.134 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,8388608~8388608,54525952~4194304,113246208~4194304,146800640~4194304,159383552~4194304,201326592~4194304,209715200~8388608,239075328~4194304,247463936~4194304,281018368~8388608,293601280~8388608,310378496~8388608,364904448~16777216,385875968~12582912] 2024-07-28T19:48:51.139 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,54525952~4194304,113246208~4194304,146800640~4194304,159383552~8388608,201326592~20971520,239075328~4194304,247463936~8388608,281018368~25165824,310378496~12582912,364904448~37748736] 2024-07-28T19:48:51.139 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 14 diff [0~4194304,8388608~8388608,54525952~4194304,75497472~4194304,113246208~4194304,146800640~4194304,159383552~4194304,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~4194304,281018368~8388608,293601280~12582912,310378496~12582912,331350016~8388608,364904448~16777216,385875968~16777216,406847488~4194304] 2024-07-28T19:48:51.139 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,8388608~8388608,54525952~4194304,75497472~4194304,113246208~4194304,146800640~4194304,159383552~4194304,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~4194304,281018368~8388608,293601280~12582912,310378496~12582912,331350016~8388608,364904448~16777216,385875968~16777216,406847488~4194304] 2024-07-28T19:48:51.145 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~16777216,54525952~4194304,75497472~8388608,113246208~4194304,146800640~4194304,159383552~8388608,201326592~20971520,226492416~8388608,239075328~4194304,247463936~8388608,281018368~25165824,310378496~16777216,331350016~12582912,364904448~46137344] 2024-07-28T19:48:51.145 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 15 diff [0~4194304,8388608~16777216,29360128~4194304,41943040~4194304,54525952~4194304,67108864~16777216,113246208~4194304,146800640~4194304,159383552~16777216,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,281018368~8388608,293601280~12582912,310378496~12582912,331350016~8388608,364904448~16777216,385875968~16777216,406847488~4194304] 2024-07-28T19:48:51.145 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~4194304,8388608~16777216,29360128~4194304,41943040~4194304,54525952~4194304,67108864~16777216,113246208~4194304,146800640~4194304,159383552~16777216,201326592~4194304,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,281018368~8388608,293601280~12582912,310378496~12582912,331350016~8388608,364904448~16777216,385875968~16777216,406847488~4194304] 2024-07-28T19:48:51.151 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~4194304,54525952~4194304,67108864~16777216,113246208~4194304,146800640~4194304,159383552~16777216,201326592~20971520,226492416~8388608,239075328~4194304,247463936~12582912,281018368~46137344,331350016~12582912,364904448~46137344] 2024-07-28T19:48:51.151 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 16 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,109051904~4194304,146800640~4194304,159383552~16777216,197132288~8388608,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~12582912,310378496~12582912,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.151 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,109051904~4194304,146800640~4194304,159383552~16777216,197132288~8388608,209715200~8388608,226492416~4194304,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~12582912,310378496~12582912,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.158 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~16777216,88080384~16777216,109051904~8388608,146800640~4194304,159383552~16777216,197132288~25165824,226492416~8388608,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~50331648] 2024-07-28T19:48:51.158 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 17 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,104857600~8388608,134217728~4194304,146800640~4194304,159383552~16777216,197132288~20971520,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~12582912,310378496~16777216,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.158 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,104857600~8388608,134217728~4194304,146800640~4194304,159383552~16777216,197132288~20971520,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~12582912,310378496~16777216,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.166 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~16777216,88080384~29360128,134217728~8388608,146800640~4194304,159383552~16777216,197132288~37748736,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~50331648] 2024-07-28T19:48:51.167 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 18 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.167 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.176 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~16777216,88080384~41943040,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:51.176 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to 19 diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.176 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.185 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:51.185 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 13 diff [54525952~4194304,146800640~4194304,159383552~4194304,201326592~4194304,247463936~4194304,281018368~8388608,310378496~8388608,385875968~8388608] 2024-07-28T19:48:51.185 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [54525952~4194304,146800640~4194304,159383552~4194304,201326592~4194304,247463936~4194304,281018368~8388608,310378496~8388608,385875968~8388608] 2024-07-28T19:48:51.188 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [54525952~4194304,146800640~4194304,159383552~8388608,201326592~8388608,247463936~8388608,281018368~12582912,310378496~12582912,385875968~12582912] 2024-07-28T19:48:51.188 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 14 diff [54525952~4194304,75497472~4194304,146800640~4194304,159383552~4194304,201326592~4194304,209715200~4194304,226492416~4194304,247463936~4194304,281018368~8388608,301989888~4194304,310378496~12582912,331350016~8388608,385875968~8388608,398458880~4194304,406847488~4194304] 2024-07-28T19:48:51.188 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [54525952~4194304,75497472~4194304,146800640~4194304,159383552~4194304,201326592~4194304,209715200~4194304,226492416~4194304,247463936~4194304,281018368~8388608,301989888~4194304,310378496~12582912,331350016~8388608,385875968~8388608,398458880~4194304,406847488~4194304] 2024-07-28T19:48:51.192 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [54525952~4194304,75497472~8388608,146800640~4194304,159383552~8388608,201326592~12582912,226492416~8388608,247463936~8388608,281018368~12582912,301989888~4194304,310378496~16777216,331350016~12582912,385875968~25165824] 2024-07-28T19:48:51.192 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 15 diff [16777216~8388608,29360128~4194304,41943040~4194304,54525952~4194304,67108864~16777216,146800640~4194304,159383552~16777216,201326592~4194304,209715200~4194304,226492416~4194304,247463936~12582912,281018368~8388608,301989888~4194304,310378496~12582912,331350016~8388608,373293056~4194304,385875968~8388608,398458880~4194304,406847488~4194304] 2024-07-28T19:48:51.192 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [16777216~8388608,29360128~4194304,41943040~4194304,54525952~4194304,67108864~16777216,146800640~4194304,159383552~16777216,201326592~4194304,209715200~4194304,226492416~4194304,247463936~12582912,281018368~8388608,301989888~4194304,310378496~12582912,331350016~8388608,373293056~4194304,385875968~8388608,398458880~4194304,406847488~4194304] 2024-07-28T19:48:51.197 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [16777216~8388608,29360128~8388608,41943040~4194304,54525952~4194304,67108864~16777216,146800640~4194304,159383552~16777216,201326592~12582912,226492416~8388608,247463936~12582912,281018368~12582912,301989888~25165824,331350016~12582912,373293056~4194304,385875968~25165824] 2024-07-28T19:48:51.198 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 16 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,109051904~4194304,146800640~4194304,159383552~16777216,197132288~8388608,209715200~4194304,226492416~4194304,247463936~12582912,272629760~4194304,281018368~8388608,301989888~4194304,310378496~12582912,331350016~8388608,360710144~4194304,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.198 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,109051904~4194304,146800640~4194304,159383552~16777216,197132288~8388608,209715200~4194304,226492416~4194304,247463936~12582912,272629760~4194304,281018368~8388608,301989888~4194304,310378496~12582912,331350016~8388608,360710144~4194304,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.204 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~16777216,88080384~16777216,109051904~8388608,146800640~4194304,159383552~16777216,197132288~16777216,226492416~8388608,247463936~12582912,272629760~20971520,301989888~25165824,331350016~12582912,360710144~4194304,373293056~4194304,385875968~25165824] 2024-07-28T19:48:51.204 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 17 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,104857600~8388608,134217728~4194304,146800640~4194304,159383552~16777216,197132288~16777216,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,301989888~4194304,310378496~16777216,331350016~8388608,360710144~4194304,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.204 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,104857600~8388608,134217728~4194304,146800640~4194304,159383552~16777216,197132288~16777216,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,301989888~4194304,310378496~16777216,331350016~8388608,360710144~4194304,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.211 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~16777216,88080384~29360128,134217728~8388608,146800640~4194304,159383552~16777216,197132288~16777216,222298112~12582912,247463936~12582912,272629760~20971520,301989888~25165824,331350016~12582912,360710144~4194304,373293056~4194304,385875968~25165824] 2024-07-28T19:48:51.211 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 18 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~25165824,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,301989888~25165824,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.211 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~16777216,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~25165824,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,301989888~25165824,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.218 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~16777216,88080384~41943040,134217728~8388608,146800640~4194304,159383552~16777216,188743680~25165824,222298112~12582912,247463936~12582912,272629760~20971520,301989888~25165824,331350016~12582912,360710144~4194304,373293056~41943040] 2024-07-28T19:48:51.218 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to 19 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:51.219 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:51.227 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~29360128,222298112~12582912,247463936~12582912,272629760~20971520,297795584~29360128,331350016~12582912,360710144~4194304,373293056~41943040] 2024-07-28T19:48:51.227 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 14 diff [75497472~4194304,209715200~4194304,226492416~4194304,301989888~4194304,318767104~8388608,331350016~8388608,385875968~4194304,398458880~4194304,406847488~4194304] 2024-07-28T19:48:51.227 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [75497472~4194304,209715200~4194304,226492416~4194304,301989888~4194304,318767104~8388608,331350016~8388608,385875968~4194304,398458880~4194304,406847488~4194304] 2024-07-28T19:48:51.230 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [75497472~8388608,209715200~4194304,226492416~8388608,301989888~4194304,318767104~8388608,331350016~12582912,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.230 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 15 diff [16777216~8388608,29360128~4194304,41943040~4194304,67108864~16777216,159383552~4194304,167772160~8388608,209715200~4194304,226492416~4194304,247463936~8388608,301989888~4194304,318767104~8388608,331350016~8388608,373293056~4194304,385875968~4194304,398458880~4194304,406847488~4194304] 2024-07-28T19:48:51.230 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [16777216~8388608,29360128~4194304,41943040~4194304,67108864~16777216,159383552~4194304,167772160~8388608,209715200~4194304,226492416~4194304,247463936~8388608,301989888~4194304,318767104~8388608,331350016~8388608,373293056~4194304,385875968~4194304,398458880~4194304,406847488~4194304] 2024-07-28T19:48:51.234 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [16777216~8388608,29360128~8388608,41943040~4194304,67108864~16777216,159383552~4194304,167772160~8388608,209715200~4194304,226492416~8388608,247463936~12582912,301989888~8388608,318767104~8388608,331350016~12582912,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.234 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 16 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,109051904~4194304,159383552~4194304,167772160~8388608,197132288~4194304,209715200~4194304,226492416~4194304,247463936~8388608,272629760~4194304,301989888~4194304,318767104~8388608,331350016~8388608,360710144~4194304,373293056~4194304,385875968~4194304,398458880~12582912] 2024-07-28T19:48:51.234 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,109051904~4194304,159383552~4194304,167772160~8388608,197132288~4194304,209715200~4194304,226492416~4194304,247463936~8388608,272629760~4194304,301989888~4194304,318767104~8388608,331350016~8388608,360710144~4194304,373293056~4194304,385875968~4194304,398458880~12582912] 2024-07-28T19:48:51.239 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~16777216,88080384~16777216,109051904~8388608,159383552~4194304,167772160~8388608,197132288~4194304,209715200~4194304,226492416~8388608,247463936~12582912,272629760~8388608,301989888~8388608,318767104~8388608,331350016~12582912,360710144~4194304,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.239 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 17 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,104857600~8388608,134217728~4194304,159383552~4194304,167772160~8388608,197132288~8388608,209715200~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,301989888~4194304,310378496~16777216,331350016~8388608,360710144~4194304,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.239 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,104857600~8388608,134217728~4194304,159383552~4194304,167772160~8388608,197132288~8388608,209715200~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,301989888~4194304,310378496~16777216,331350016~8388608,360710144~4194304,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.244 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~16777216,88080384~29360128,134217728~8388608,159383552~4194304,167772160~8388608,197132288~16777216,222298112~12582912,247463936~12582912,272629760~8388608,285212672~4194304,301989888~25165824,331350016~12582912,360710144~4194304,373293056~4194304,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.244 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 18 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~16777216,209715200~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,301989888~25165824,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.245 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~16777216,209715200~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,301989888~25165824,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.251 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~16777216,88080384~41943040,134217728~8388608,159383552~4194304,167772160~8388608,188743680~25165824,222298112~12582912,247463936~12582912,272629760~8388608,285212672~4194304,301989888~25165824,331350016~12582912,360710144~4194304,373293056~20971520,398458880~16777216] 2024-07-28T19:48:51.251 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to 19 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.251 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.259 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~12582912,247463936~12582912,272629760~8388608,285212672~4194304,297795584~29360128,331350016~12582912,360710144~4194304,373293056~20971520,398458880~16777216] 2024-07-28T19:48:51.259 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 15 diff [16777216~8388608,29360128~4194304,41943040~4194304,67108864~16777216,159383552~4194304,167772160~8388608,247463936~8388608,301989888~4194304,373293056~4194304] 2024-07-28T19:48:51.259 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [16777216~8388608,29360128~4194304,41943040~4194304,67108864~16777216,159383552~4194304,167772160~8388608,247463936~8388608,301989888~4194304,373293056~4194304] 2024-07-28T19:48:51.262 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [16777216~8388608,29360128~8388608,41943040~4194304,67108864~16777216,159383552~4194304,167772160~8388608,247463936~12582912,301989888~8388608,373293056~4194304] 2024-07-28T19:48:51.262 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 16 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,109051904~4194304,159383552~4194304,167772160~8388608,197132288~4194304,247463936~8388608,272629760~4194304,301989888~4194304,360710144~4194304,373293056~4194304,402653184~8388608] 2024-07-28T19:48:51.262 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,109051904~4194304,159383552~4194304,167772160~8388608,197132288~4194304,247463936~8388608,272629760~4194304,301989888~4194304,360710144~4194304,373293056~4194304,402653184~8388608] 2024-07-28T19:48:51.265 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~16777216,88080384~16777216,109051904~8388608,159383552~4194304,167772160~8388608,197132288~4194304,247463936~12582912,272629760~8388608,301989888~8388608,360710144~4194304,373293056~4194304,402653184~8388608] 2024-07-28T19:48:51.266 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 17 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,104857600~8388608,134217728~4194304,159383552~4194304,167772160~8388608,197132288~8388608,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,301989888~4194304,310378496~8388608,360710144~4194304,373293056~4194304,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.266 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,104857600~8388608,134217728~4194304,159383552~4194304,167772160~8388608,197132288~8388608,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,301989888~4194304,310378496~8388608,360710144~4194304,373293056~4194304,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.271 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~16777216,88080384~29360128,134217728~8388608,159383552~4194304,167772160~8388608,197132288~12582912,222298112~8388608,247463936~12582912,272629760~8388608,285212672~4194304,301989888~16777216,360710144~4194304,373293056~4194304,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.271 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 18 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~16777216,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,301989888~16777216,360710144~4194304,373293056~8388608,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.271 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~16777216,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~16777216,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,301989888~16777216,360710144~4194304,373293056~8388608,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.277 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~16777216,88080384~41943040,134217728~8388608,159383552~4194304,167772160~8388608,188743680~20971520,222298112~8388608,247463936~12582912,272629760~8388608,285212672~4194304,301989888~16777216,360710144~4194304,373293056~16777216,402653184~12582912] 2024-07-28T19:48:51.277 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to 19 diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.277 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.284 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~12582912,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:51.284 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to 16 diff [4194304~8388608,41943040~8388608,71303168~4194304,88080384~12582912,109051904~4194304,197132288~4194304,272629760~4194304,360710144~4194304,402653184~8388608] 2024-07-28T19:48:51.284 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,41943040~8388608,71303168~4194304,88080384~12582912,109051904~4194304,197132288~4194304,272629760~4194304,360710144~4194304,402653184~8388608] 2024-07-28T19:48:51.287 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~12582912,41943040~12582912,71303168~4194304,88080384~16777216,109051904~8388608,197132288~4194304,272629760~8388608,360710144~4194304,402653184~8388608] 2024-07-28T19:48:51.287 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to 17 diff [4194304~8388608,41943040~8388608,71303168~4194304,88080384~12582912,104857600~8388608,134217728~4194304,197132288~8388608,222298112~8388608,272629760~4194304,285212672~4194304,310378496~8388608,360710144~4194304,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.287 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,41943040~8388608,71303168~4194304,88080384~12582912,104857600~8388608,134217728~4194304,197132288~8388608,222298112~8388608,272629760~4194304,285212672~4194304,310378496~8388608,360710144~4194304,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.291 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~12582912,41943040~12582912,71303168~4194304,88080384~29360128,134217728~8388608,197132288~12582912,222298112~8388608,272629760~8388608,285212672~4194304,310378496~8388608,360710144~4194304,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.291 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to 18 diff [4194304~8388608,41943040~8388608,67108864~8388608,79691776~4194304,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,188743680~16777216,222298112~8388608,272629760~4194304,285212672~4194304,301989888~16777216,360710144~4194304,373293056~8388608,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.292 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,41943040~8388608,67108864~8388608,79691776~4194304,88080384~12582912,104857600~8388608,117440512~8388608,134217728~4194304,188743680~16777216,222298112~8388608,272629760~4194304,285212672~4194304,301989888~16777216,360710144~4194304,373293056~8388608,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.297 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~12582912,41943040~12582912,67108864~8388608,79691776~4194304,88080384~41943040,134217728~8388608,188743680~20971520,222298112~8388608,272629760~8388608,285212672~4194304,301989888~16777216,360710144~4194304,373293056~16777216,402653184~12582912] 2024-07-28T19:48:51.297 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to 19 diff [4194304~8388608,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.297 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.303 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~12582912,41943040~12582912,67108864~62914560,134217728~8388608,188743680~29360128,222298112~8388608,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:51.303 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to 17 diff [104857600~8388608,134217728~4194304,197132288~8388608,222298112~8388608,285212672~4194304,310378496~8388608,385875968~4194304,402653184~4194304] 2024-07-28T19:48:51.304 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [104857600~8388608,134217728~4194304,197132288~8388608,222298112~8388608,285212672~4194304,310378496~8388608,385875968~4194304,402653184~4194304] 2024-07-28T19:48:51.306 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [104857600~12582912,134217728~8388608,197132288~12582912,222298112~8388608,285212672~4194304,310378496~8388608,385875968~4194304,402653184~4194304] 2024-07-28T19:48:51.306 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to 18 diff [41943040~4194304,67108864~8388608,79691776~4194304,104857600~8388608,117440512~8388608,134217728~4194304,188743680~16777216,222298112~8388608,285212672~4194304,301989888~16777216,373293056~8388608,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.306 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [41943040~4194304,67108864~8388608,79691776~4194304,104857600~8388608,117440512~8388608,134217728~4194304,188743680~16777216,222298112~8388608,285212672~4194304,301989888~16777216,373293056~8388608,385875968~4194304,402653184~8388608] 2024-07-28T19:48:51.310 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [41943040~4194304,67108864~8388608,79691776~4194304,104857600~25165824,134217728~8388608,188743680~20971520,222298112~8388608,285212672~4194304,301989888~16777216,373293056~16777216,402653184~12582912] 2024-07-28T19:48:51.310 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to 19 diff [8388608~4194304,41943040~4194304,67108864~20971520,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.310 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~4194304,41943040~4194304,67108864~20971520,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.315 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4194304,41943040~4194304,67108864~20971520,104857600~25165824,134217728~8388608,188743680~29360128,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~20971520,402653184~12582912] 2024-07-28T19:48:51.315 INFO:tasks.workunit.client.0.smithi100.stdout:from 17 to 18 diff [41943040~4194304,67108864~8388608,79691776~4194304,117440512~8388608,188743680~8388608,301989888~8388608,314572800~4194304,373293056~8388608,402653184~8388608] 2024-07-28T19:48:51.315 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [41943040~4194304,67108864~8388608,79691776~4194304,117440512~8388608,188743680~8388608,301989888~8388608,314572800~4194304,373293056~8388608,402653184~8388608] 2024-07-28T19:48:51.318 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [41943040~4194304,67108864~8388608,79691776~4194304,117440512~12582912,188743680~8388608,301989888~8388608,314572800~4194304,373293056~12582912,402653184~12582912] 2024-07-28T19:48:51.318 INFO:tasks.workunit.client.0.smithi100.stdout:from 17 to 19 diff [8388608~4194304,41943040~4194304,67108864~20971520,117440512~8388608,188743680~8388608,205520896~4194304,213909504~4194304,222298112~4194304,297795584~12582912,314572800~4194304,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.318 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~4194304,41943040~4194304,67108864~20971520,117440512~8388608,188743680~8388608,205520896~4194304,213909504~4194304,222298112~4194304,297795584~12582912,314572800~4194304,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.322 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4194304,41943040~4194304,67108864~20971520,117440512~12582912,188743680~8388608,205520896~12582912,222298112~4194304,297795584~12582912,314572800~4194304,331350016~8388608,373293056~20971520,402653184~12582912] 2024-07-28T19:48:51.322 INFO:tasks.workunit.client.0.smithi100.stdout:from 18 to 19 diff [8388608~4194304,75497472~12582912,205520896~4194304,213909504~4194304,222298112~4194304,297795584~4194304,331350016~8388608,385875968~8388608] 2024-07-28T19:48:51.322 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~4194304,75497472~12582912,205520896~4194304,213909504~4194304,222298112~4194304,297795584~4194304,331350016~8388608,385875968~8388608] 2024-07-28T19:48:51.324 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4194304,75497472~12582912,205520896~12582912,222298112~4194304,297795584~4194304,331350016~8388608,385875968~8388608] 2024-07-28T19:48:51.517 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:51.517 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:51.551 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:51.551 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:51.551 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:51.566 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:51.566 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:51.566 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:51.581 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:51.581 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:51.582 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:51.596 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:51.596 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:51.596 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:51.611 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:51.611 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:51.611 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:51.624 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:51.624 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:51.624 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:51.636 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:51.636 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:51.637 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:51.650 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:51.650 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.650 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.659 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:51.659 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.659 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.668 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:51.669 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.669 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:51.677 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:51.678 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.678 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:51.687 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:51.687 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:51.687 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:51.693 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~29360128,222298112~12582912,247463936~12582912,272629760~20971520,297795584~29360128,331350016~12582912,360710144~4194304,373293056~41943040] 2024-07-28T19:48:51.693 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.693 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:51.699 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~12582912,247463936~12582912,272629760~8388608,285212672~4194304,297795584~29360128,331350016~12582912,360710144~4194304,373293056~20971520,398458880~16777216] 2024-07-28T19:48:51.699 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.699 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.704 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~12582912,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:51.704 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to HEAD diff [4194304~8388608,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.704 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.708 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~12582912,41943040~12582912,67108864~62914560,134217728~8388608,188743680~29360128,222298112~8388608,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:51.708 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to HEAD diff [8388608~4194304,41943040~4194304,67108864~20971520,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.708 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~4194304,41943040~4194304,67108864~20971520,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.712 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4194304,41943040~4194304,67108864~20971520,104857600~25165824,134217728~8388608,188743680~29360128,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~20971520,402653184~12582912] 2024-07-28T19:48:51.712 INFO:tasks.workunit.client.0.smithi100.stdout:from 17 to HEAD diff [8388608~4194304,41943040~4194304,67108864~20971520,117440512~8388608,188743680~8388608,205520896~4194304,213909504~4194304,222298112~4194304,297795584~12582912,314572800~4194304,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.712 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~4194304,41943040~4194304,67108864~20971520,117440512~8388608,188743680~8388608,205520896~4194304,213909504~4194304,222298112~4194304,297795584~12582912,314572800~4194304,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:51.714 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4194304,41943040~4194304,67108864~20971520,117440512~12582912,188743680~8388608,205520896~12582912,222298112~4194304,297795584~12582912,314572800~4194304,331350016~8388608,373293056~20971520,402653184~12582912] 2024-07-28T19:48:52.540 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:52.540 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:52.578 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:52.578 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:52.578 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:52.595 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:52.595 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:52.596 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:52.612 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:52.613 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:52.613 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:52.629 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:52.629 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:52.629 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:52.644 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:52.644 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:52.644 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:52.658 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:52.658 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:52.658 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:52.670 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:52.670 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:52.670 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:52.682 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:52.683 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:52.683 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:52.693 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:52.693 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:52.693 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:52.703 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:52.703 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:52.704 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:52.712 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:52.712 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:52.712 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:52.720 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:52.720 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:52.720 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:52.727 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~29360128,222298112~12582912,247463936~12582912,272629760~20971520,297795584~29360128,331350016~12582912,360710144~4194304,373293056~41943040] 2024-07-28T19:48:52.727 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:52.727 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:52.733 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~12582912,247463936~12582912,272629760~8388608,285212672~4194304,297795584~29360128,331350016~12582912,360710144~4194304,373293056~20971520,398458880~16777216] 2024-07-28T19:48:52.733 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:52.733 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:52.738 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~12582912,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:52.738 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to HEAD diff [4194304~8388608,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:52.738 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:52.743 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~12582912,41943040~12582912,67108864~62914560,134217728~8388608,188743680~29360128,222298112~8388608,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:52.743 INFO:tasks.workunit.client.0.smithi100.stdout:from 16 to HEAD diff [8388608~4194304,41943040~4194304,67108864~20971520,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:52.743 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [8388608~4194304,41943040~4194304,67108864~20971520,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:52.745 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [8388608~4194304,41943040~4194304,67108864~20971520,104857600~25165824,134217728~8388608,188743680~29360128,222298112~8388608,285212672~4194304,297795584~20971520,331350016~8388608,373293056~20971520,402653184~12582912] 2024-07-28T19:48:52.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:52 smithi138 bash[67894]: cluster 2024-07-28T19:48:51.502803+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-07-28T19:48:52.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:52 smithi138 bash[67894]: cluster 2024-07-28T19:48:52.065205+0000 mgr.y (mgr.25358) 488 : cluster [DBG] pgmap v1034: 105 pgs: 105 active+clean; 834 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 31 MiB/s wr, 365 op/s 2024-07-28T19:48:52.808 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:48:52 smithi138 bash[60615]: ts=2024-07-28T19:48:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:48:52.861 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:52 smithi100 bash[82644]: cluster 2024-07-28T19:48:51.502803+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-07-28T19:48:52.861 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:52 smithi100 bash[82644]: cluster 2024-07-28T19:48:52.065205+0000 mgr.y (mgr.25358) 488 : cluster [DBG] pgmap v1034: 105 pgs: 105 active+clean; 834 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 31 MiB/s wr, 365 op/s 2024-07-28T19:48:52.862 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:52 smithi100 bash[80199]: cluster 2024-07-28T19:48:51.502803+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-07-28T19:48:52.862 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:52 smithi100 bash[80199]: cluster 2024-07-28T19:48:52.065205+0000 mgr.y (mgr.25358) 488 : cluster [DBG] pgmap v1034: 105 pgs: 105 active+clean; 834 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 31 MiB/s wr, 365 op/s 2024-07-28T19:48:53.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:48:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:48:52] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2024-07-28T19:48:53.554 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:53.554 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:53.589 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:53.589 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:53.589 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:53.604 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:53.604 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:53.604 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:53.618 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:53.618 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:53.618 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:53.632 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:53.632 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:53.633 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:53.645 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:53.645 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:53.645 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:53.658 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:53.658 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:53.658 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:53.670 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:53.671 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:53.671 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:53.682 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:53.682 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:53.682 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:53.693 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:53.694 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:53.694 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:53.703 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:53.703 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:53.703 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:53.712 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:53.713 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:53.713 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:53.720 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:53.720 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:53.720 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:53.726 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~29360128,222298112~12582912,247463936~12582912,272629760~20971520,297795584~29360128,331350016~12582912,360710144~4194304,373293056~41943040] 2024-07-28T19:48:53.726 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:53.726 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:53.732 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~12582912,247463936~12582912,272629760~8388608,285212672~4194304,297795584~29360128,331350016~12582912,360710144~4194304,373293056~20971520,398458880~16777216] 2024-07-28T19:48:53.732 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:53.732 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:53.736 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~12582912,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:53.736 INFO:tasks.workunit.client.0.smithi100.stdout:from 15 to HEAD diff [4194304~8388608,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:53.736 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,188743680~20971520,213909504~4194304,222298112~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:53.740 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~12582912,41943040~12582912,67108864~62914560,134217728~8388608,188743680~29360128,222298112~8388608,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:53 smithi138 bash[67894]: cluster 2024-07-28T19:48:52.513567+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-07-28T19:48:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:53 smithi138 bash[67894]: audit 2024-07-28T19:48:52.644729+0000 mon.a (mon.0) 1501 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:53.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:53 smithi100 bash[80199]: cluster 2024-07-28T19:48:52.513567+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-07-28T19:48:53.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:53 smithi100 bash[80199]: audit 2024-07-28T19:48:52.644729+0000 mon.a (mon.0) 1501 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:53.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:53 smithi100 bash[82644]: cluster 2024-07-28T19:48:52.513567+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-07-28T19:48:53.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:53 smithi100 bash[82644]: audit 2024-07-28T19:48:52.644729+0000 mon.a (mon.0) 1501 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:48:54.573 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:54.574 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:54.607 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:54.607 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:54.607 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:54.620 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:54.620 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:54.620 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:54.635 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:54.635 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:54.635 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:54.649 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:54.649 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:54.649 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:54.662 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:54.662 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:54.662 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:54.675 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:54.675 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:54.675 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:54.687 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:54.687 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:54.687 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:54.699 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:54.699 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:54.699 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:54.709 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:54.709 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:54.709 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:54.718 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:54.718 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:54.718 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:54.726 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:54.726 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:54.726 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:54.733 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:54.733 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:54.733 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:54.739 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~29360128,222298112~12582912,247463936~12582912,272629760~20971520,297795584~29360128,331350016~12582912,360710144~4194304,373293056~41943040] 2024-07-28T19:48:54.739 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:54.739 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:54.743 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~12582912,247463936~12582912,272629760~8388608,285212672~4194304,297795584~29360128,331350016~12582912,360710144~4194304,373293056~20971520,398458880~16777216] 2024-07-28T19:48:54.743 INFO:tasks.workunit.client.0.smithi100.stdout:from 14 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:54.743 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~20971520,213909504~4194304,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,402653184~8388608] 2024-07-28T19:48:54.746 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~12582912,272629760~8388608,285212672~4194304,297795584~20971520,331350016~8388608,360710144~4194304,373293056~20971520,402653184~12582912] 2024-07-28T19:48:54.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:54 smithi138 bash[67894]: cluster 2024-07-28T19:48:53.526833+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-07-28T19:48:54.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:54 smithi138 bash[67894]: cluster 2024-07-28T19:48:54.066128+0000 mgr.y (mgr.25358) 489 : cluster [DBG] pgmap v1037: 105 pgs: 105 active+clean; 845 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 294 KiB/s rd, 15 MiB/s wr, 528 op/s 2024-07-28T19:48:54.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:54 smithi100 bash[80199]: cluster 2024-07-28T19:48:53.526833+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-07-28T19:48:54.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:54 smithi100 bash[80199]: cluster 2024-07-28T19:48:54.066128+0000 mgr.y (mgr.25358) 489 : cluster [DBG] pgmap v1037: 105 pgs: 105 active+clean; 845 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 294 KiB/s rd, 15 MiB/s wr, 528 op/s 2024-07-28T19:48:54.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:54 smithi100 bash[82644]: cluster 2024-07-28T19:48:53.526833+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-07-28T19:48:54.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:54 smithi100 bash[82644]: cluster 2024-07-28T19:48:54.066128+0000 mgr.y (mgr.25358) 489 : cluster [DBG] pgmap v1037: 105 pgs: 105 active+clean; 845 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 294 KiB/s rd, 15 MiB/s wr, 528 op/s 2024-07-28T19:48:55.566 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:55.566 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:55.606 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:55.606 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:55.606 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:55.616 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:55.616 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:55.616 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:55.626 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:55.626 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:55.626 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:55.635 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:55.635 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:55.635 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:55.644 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:55.644 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:55.644 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:55.651 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:55.651 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:55.651 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:55.658 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:55.659 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:55.659 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:55.665 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:55.665 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:55.666 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:55.672 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:55.672 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:55.673 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:55.678 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:55.678 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:55.678 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:55.683 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:55.683 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:55.683 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:55.687 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:55.687 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:55.687 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:55.691 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~29360128,222298112~12582912,247463936~12582912,272629760~20971520,297795584~29360128,331350016~12582912,360710144~4194304,373293056~41943040] 2024-07-28T19:48:55.691 INFO:tasks.workunit.client.0.smithi100.stdout:from 13 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:55.691 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,159383552~4194304,167772160~8388608,188743680~29360128,222298112~8388608,247463936~8388608,272629760~4194304,285212672~4194304,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~8388608,398458880~12582912] 2024-07-28T19:48:55.693 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~12582912,67108864~62914560,134217728~8388608,159383552~4194304,167772160~8388608,188743680~29360128,222298112~12582912,247463936~12582912,272629760~8388608,285212672~4194304,297795584~29360128,331350016~12582912,360710144~4194304,373293056~20971520,398458880~16777216] 2024-07-28T19:48:55.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:55 smithi138 bash[67894]: cluster 2024-07-28T19:48:54.538601+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-07-28T19:48:55.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:55 smithi100 bash[80199]: cluster 2024-07-28T19:48:54.538601+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-07-28T19:48:55.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:55 smithi100 bash[82644]: cluster 2024-07-28T19:48:54.538601+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-07-28T19:48:56.584 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:56.584 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:56.636 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:56.636 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:56.636 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:56.650 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:56.650 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:56.650 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:56.657 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:56.657 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:56.657 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:56.666 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:56.666 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:56.666 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:56.673 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:56.673 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:56.673 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:56.681 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:56.681 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:56.681 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:56.688 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:56.689 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:56.689 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:56.695 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:56.695 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:56.695 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:56.702 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:56.702 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:56.702 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:56.709 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:56.709 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:56.709 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:56.714 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:56.714 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:56.714 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:56.718 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:56.718 INFO:tasks.workunit.client.0.smithi100.stdout:from 12 to HEAD diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:56.719 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [4194304~8388608,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,247463936~12582912,272629760~4194304,281018368~8388608,297795584~29360128,331350016~8388608,360710144~4194304,373293056~8388608,385875968~25165824] 2024-07-28T19:48:56.721 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [4194304~20971520,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~29360128,222298112~12582912,247463936~12582912,272629760~20971520,297795584~29360128,331350016~12582912,360710144~4194304,373293056~41943040] 2024-07-28T19:48:56.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:56 smithi100 bash[80199]: cluster 2024-07-28T19:48:55.539362+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-07-28T19:48:56.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:56 smithi100 bash[80199]: cluster 2024-07-28T19:48:56.066776+0000 mgr.y (mgr.25358) 490 : cluster [DBG] pgmap v1040: 105 pgs: 105 active+clean; 842 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 663 KiB/s rd, 3.8 MiB/s wr, 1.25k op/s 2024-07-28T19:48:56.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:56 smithi100 bash[82644]: cluster 2024-07-28T19:48:55.539362+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-07-28T19:48:56.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:56 smithi100 bash[82644]: cluster 2024-07-28T19:48:56.066776+0000 mgr.y (mgr.25358) 490 : cluster [DBG] pgmap v1040: 105 pgs: 105 active+clean; 842 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 663 KiB/s rd, 3.8 MiB/s wr, 1.25k op/s 2024-07-28T19:48:57.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:56 smithi138 bash[67894]: cluster 2024-07-28T19:48:55.539362+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-07-28T19:48:57.058 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:56 smithi138 bash[67894]: cluster 2024-07-28T19:48:56.066776+0000 mgr.y (mgr.25358) 490 : cluster [DBG] pgmap v1040: 105 pgs: 105 active+clean; 842 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 663 KiB/s rd, 3.8 MiB/s wr, 1.25k op/s 2024-07-28T19:48:57.603 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:57.603 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:57.638 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:57.638 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:57.638 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:57.646 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:57.646 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:57.646 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:57.655 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:57.655 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:57.655 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:57.663 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:57.663 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:57.663 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:57.670 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:57.670 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:57.671 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:57.677 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:57.677 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:57.677 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:57.684 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:57.684 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:57.684 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:57.691 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:57.691 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:57.691 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:57.698 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:57.698 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:57.698 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:57.704 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:57.704 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:57.704 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:57.709 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:57.709 INFO:tasks.workunit.client.0.smithi100.stdout:from 11 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:57.709 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~8388608,54525952~4194304,67108864~33554432,104857600~8388608,117440512~8388608,134217728~4194304,146800640~4194304,159383552~16777216,188743680~29360128,222298112~8388608,239075328~4194304,247463936~12582912,272629760~4194304,281018368~8388608,293601280~33554432,331350016~8388608,360710144~20971520,385875968~25165824] 2024-07-28T19:48:57.713 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~16777216,188743680~46137344,239075328~4194304,247463936~12582912,272629760~54525952,331350016~12582912,360710144~54525952] 2024-07-28T19:48:57.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:57 smithi100 bash[80199]: cluster 2024-07-28T19:48:56.564169+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-07-28T19:48:57.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:57 smithi100 bash[82644]: cluster 2024-07-28T19:48:56.564169+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-07-28T19:48:58.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:57 smithi138 bash[67894]: cluster 2024-07-28T19:48:56.564169+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-07-28T19:48:58.614 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:58.614 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:58.644 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:58.644 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:58.644 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:58.655 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:58.655 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:58.655 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:58.663 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:58.663 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:58.663 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:58.668 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:58.669 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:58.669 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:58.675 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:58.675 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:58.675 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:58.680 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:58.680 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:58.680 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:58.685 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:58.685 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:58.685 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:58.690 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:58.690 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:58.690 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:58.693 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:58.693 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:58.693 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:58.696 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:58.696 INFO:tasks.workunit.client.0.smithi100.stdout:from 10 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:58.696 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~16777216,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~46137344,343932928~16777216,364904448~16777216,385875968~25165824] 2024-07-28T19:48:58.698 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~16777216,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:58.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:58 smithi100 bash[80199]: cluster 2024-07-28T19:48:57.570881+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-07-28T19:48:58.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:58 smithi100 bash[80199]: cluster 2024-07-28T19:48:58.067447+0000 mgr.y (mgr.25358) 491 : cluster [DBG] pgmap v1043: 105 pgs: 105 active+clean; 839 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 360 KiB/s rd, 255 B/s wr, 719 op/s 2024-07-28T19:48:58.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:58 smithi100 bash[82644]: cluster 2024-07-28T19:48:57.570881+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-07-28T19:48:58.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:58 smithi100 bash[82644]: cluster 2024-07-28T19:48:58.067447+0000 mgr.y (mgr.25358) 491 : cluster [DBG] pgmap v1043: 105 pgs: 105 active+clean; 839 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 360 KiB/s rd, 255 B/s wr, 719 op/s 2024-07-28T19:48:59.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:58 smithi138 bash[67894]: cluster 2024-07-28T19:48:57.570881+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-07-28T19:48:59.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:58 smithi138 bash[67894]: cluster 2024-07-28T19:48:58.067447+0000 mgr.y (mgr.25358) 491 : cluster [DBG] pgmap v1043: 105 pgs: 105 active+clean; 839 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 360 KiB/s rd, 255 B/s wr, 719 op/s 2024-07-28T19:48:59.605 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:59.605 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:48:59.654 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:48:59.654 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:59.654 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:48:59.663 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:59.663 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:59.663 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:59.670 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:59.670 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:59.670 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:59.676 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:48:59.676 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:59.676 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:48:59.681 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:48:59.681 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:59.681 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:59.685 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:59.685 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:59.686 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:59.690 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:48:59.690 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:59.690 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:48:59.695 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:59.695 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:59.695 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:48:59.698 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:48:59.698 INFO:tasks.workunit.client.0.smithi100.stdout:from 9 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:59.698 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~8388608,134217728~4194304,146800640~4194304,159383552~58720256,222298112~8388608,239075328~4194304,247463936~12582912,264241152~12582912,281018368~8388608,293601280~58720256,356515840~4194304,364904448~16777216,385875968~25165824] 2024-07-28T19:48:59.700 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~62914560,134217728~8388608,146800640~4194304,159383552~75497472,239075328~4194304,247463936~12582912,264241152~150994944] 2024-07-28T19:48:59.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:48:59 smithi100 bash[80199]: cluster 2024-07-28T19:48:58.581448+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-07-28T19:48:59.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:48:59 smithi100 bash[82644]: cluster 2024-07-28T19:48:58.581448+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-07-28T19:49:00.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:48:59 smithi138 bash[67894]: cluster 2024-07-28T19:48:58.581448+0000 mon.a (mon.0) 1507 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-07-28T19:49:00.312 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:00.312 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:00.352 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:00.353 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:00.353 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:00.364 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:00.365 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:00.365 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:00.371 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:00.371 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:00.371 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:00.376 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:00.376 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:00.376 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:00.381 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:49:00.381 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:00.381 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:00.385 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:49:00.385 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:00.385 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:00.389 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:49:00.389 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:00.389 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:00.392 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:49:00.392 INFO:tasks.workunit.client.0.smithi100.stdout:from 8 to HEAD diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:49:00.392 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~12582912,16777216~8388608,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~25165824] 2024-07-28T19:49:00.394 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:49:00.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:00 smithi100 bash[80199]: cluster 2024-07-28T19:48:59.582323+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-07-28T19:49:00.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:00 smithi100 bash[80199]: cluster 2024-07-28T19:49:00.068106+0000 mgr.y (mgr.25358) 492 : cluster [DBG] pgmap v1046: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 795 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 245 KiB/s rd, 6.2 KiB/s wr, 414 op/s 2024-07-28T19:49:00.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:00 smithi100 bash[80199]: cluster 2024-07-28T19:49:00.284214+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-07-28T19:49:00.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:00 smithi100 bash[82644]: cluster 2024-07-28T19:48:59.582323+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-07-28T19:49:00.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:00 smithi100 bash[82644]: cluster 2024-07-28T19:49:00.068106+0000 mgr.y (mgr.25358) 492 : cluster [DBG] pgmap v1046: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 795 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 245 KiB/s rd, 6.2 KiB/s wr, 414 op/s 2024-07-28T19:49:00.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:00 smithi100 bash[82644]: cluster 2024-07-28T19:49:00.284214+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-07-28T19:49:01.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:00 smithi138 bash[67894]: cluster 2024-07-28T19:48:59.582323+0000 mon.a (mon.0) 1508 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-07-28T19:49:01.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:00 smithi138 bash[67894]: cluster 2024-07-28T19:49:00.068106+0000 mgr.y (mgr.25358) 492 : cluster [DBG] pgmap v1046: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 795 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 245 KiB/s rd, 6.2 KiB/s wr, 414 op/s 2024-07-28T19:49:01.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:00 smithi138 bash[67894]: cluster 2024-07-28T19:49:00.284214+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-07-28T19:49:01.302 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:01.302 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:01.346 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:01.346 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:01.346 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:01.360 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:01.361 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:01.361 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:01.370 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:01.370 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:01.370 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:01.374 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:01.374 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:01.374 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:01.378 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:49:01.379 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:01.379 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:01.382 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:49:01.382 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:01.382 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:01.385 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:49:01.385 INFO:tasks.workunit.client.0.smithi100.stdout:from 7 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:01.386 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~4194304,146800640~4194304,159383552~58720256,222298112~54525952,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:01.387 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~75497472,146800640~4194304,159383552~255852544] 2024-07-28T19:49:02.336 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:02.336 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:02.365 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:02.365 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:02.365 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:02.369 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:02.369 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:02.369 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:02.372 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:02.372 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:02.372 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:02.375 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:02.375 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:02.376 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:02.378 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:49:02.378 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:02.378 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:02.380 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:49:02.380 INFO:tasks.workunit.client.0.smithi100.stdout:from 6 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:02.380 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:02.381 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:49:02.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:02 smithi138 bash[67894]: cluster 2024-07-28T19:49:01.287366+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-07-28T19:49:02.558 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:49:02 smithi138 bash[60615]: ts=2024-07-28T19:49:02.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:49:02.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:02 smithi100 bash[80199]: cluster 2024-07-28T19:49:01.287366+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-07-28T19:49:02.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:02 smithi100 bash[82644]: cluster 2024-07-28T19:49:01.287366+0000 mon.a (mon.0) 1510 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-07-28T19:49:03.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:49:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:49:02] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2024-07-28T19:49:03.348 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:03.348 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:03.368 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:03.368 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:03.368 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:03.375 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:03.376 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:03.376 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:03.381 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:03.381 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:03.381 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:03.386 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:03.386 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:03.386 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:03.389 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:49:03.389 INFO:tasks.workunit.client.0.smithi100.stdout:from 5 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:03.389 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,159383552~117440512,281018368~8388608,293601280~67108864,364904448~16777216,385875968~29360128] 2024-07-28T19:49:03.392 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,159383552~255852544] 2024-07-28T19:49:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:03 smithi138 bash[67894]: cluster 2024-07-28T19:49:02.068761+0000 mgr.y (mgr.25358) 493 : cluster [DBG] pgmap v1049: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 781 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 268 KiB/s rd, 6.2 KiB/s wr, 460 op/s 2024-07-28T19:49:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:03 smithi138 bash[67894]: cluster 2024-07-28T19:49:02.304209+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-07-28T19:49:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:03 smithi100 bash[80199]: cluster 2024-07-28T19:49:02.068761+0000 mgr.y (mgr.25358) 493 : cluster [DBG] pgmap v1049: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 781 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 268 KiB/s rd, 6.2 KiB/s wr, 460 op/s 2024-07-28T19:49:03.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:03 smithi100 bash[80199]: cluster 2024-07-28T19:49:02.304209+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-07-28T19:49:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:03 smithi100 bash[82644]: cluster 2024-07-28T19:49:02.068761+0000 mgr.y (mgr.25358) 493 : cluster [DBG] pgmap v1049: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 781 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 268 KiB/s rd, 6.2 KiB/s wr, 460 op/s 2024-07-28T19:49:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:03 smithi100 bash[82644]: cluster 2024-07-28T19:49:02.304209+0000 mon.a (mon.0) 1511 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-07-28T19:49:04.346 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:04.347 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:04.382 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:04.382 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:04.382 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:04.392 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:04.393 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:04.393 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:04.399 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:04.400 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:04.400 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:04.403 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:04.403 INFO:tasks.workunit.client.0.smithi100.stdout:from 4 to HEAD diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:04.403 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~4194304,41943040~20971520,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:04.406 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~8388608,41943040~20971520,67108864~83886080,155189248~260046848] 2024-07-28T19:49:04.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:04 smithi100 bash[80199]: cluster 2024-07-28T19:49:03.311941+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-07-28T19:49:04.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:04 smithi100 bash[82644]: cluster 2024-07-28T19:49:03.311941+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-07-28T19:49:04.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:04 smithi138 bash[67894]: cluster 2024-07-28T19:49:03.311941+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-07-28T19:49:05.325 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:05.325 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:05.344 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:05.344 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:05.344 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:05.350 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:05.350 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:05.350 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:05.354 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:05.355 INFO:tasks.workunit.client.0.smithi100.stdout:from 3 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:05.355 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:05.358 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:05.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:05 smithi100 bash[80199]: cluster 2024-07-28T19:49:04.069712+0000 mgr.y (mgr.25358) 494 : cluster [DBG] pgmap v1052: 105 pgs: 105 active+clean; 721 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 145 KiB/s rd, 6.5 KiB/s wr, 234 op/s 2024-07-28T19:49:05.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:05 smithi100 bash[80199]: cluster 2024-07-28T19:49:04.323659+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-07-28T19:49:05.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:05 smithi100 bash[80199]: cluster 2024-07-28T19:49:05.289245+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-07-28T19:49:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:05 smithi100 bash[82644]: cluster 2024-07-28T19:49:04.069712+0000 mgr.y (mgr.25358) 494 : cluster [DBG] pgmap v1052: 105 pgs: 105 active+clean; 721 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 145 KiB/s rd, 6.5 KiB/s wr, 234 op/s 2024-07-28T19:49:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:05 smithi100 bash[82644]: cluster 2024-07-28T19:49:04.323659+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-07-28T19:49:05.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:05 smithi100 bash[82644]: cluster 2024-07-28T19:49:05.289245+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-07-28T19:49:05.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:05 smithi138 bash[67894]: cluster 2024-07-28T19:49:04.069712+0000 mgr.y (mgr.25358) 494 : cluster [DBG] pgmap v1052: 105 pgs: 105 active+clean; 721 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 145 KiB/s rd, 6.5 KiB/s wr, 234 op/s 2024-07-28T19:49:05.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:05 smithi138 bash[67894]: cluster 2024-07-28T19:49:04.323659+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-07-28T19:49:05.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:05 smithi138 bash[67894]: cluster 2024-07-28T19:49:05.289245+0000 mon.a (mon.0) 1514 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-07-28T19:49:06.374 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:06.374 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:06.395 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:06.395 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:06.396 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:06.399 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:06.399 INFO:tasks.workunit.client.0.smithi100.stdout:from 2 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:06.400 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~8388608,293601280~67108864,364904448~50331648] 2024-07-28T19:49:06.402 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:07.378 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:07.378 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:07.406 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:07.406 INFO:tasks.workunit.client.0.smithi100.stdout:from 1 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:07.406 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~12582912,134217728~16777216,155189248~121634816,281018368~79691776,364904448~50331648] 2024-07-28T19:49:07.409 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~83886080,155189248~260046848] 2024-07-28T19:49:07.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:07 smithi100 bash[80199]: cluster 2024-07-28T19:49:06.070392+0000 mgr.y (mgr.25358) 495 : cluster [DBG] pgmap v1055: 105 pgs: 105 active+clean; 666 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 156 KiB/s rd, 6.5 KiB/s wr, 257 op/s 2024-07-28T19:49:07.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:07 smithi100 bash[80199]: cluster 2024-07-28T19:49:06.333954+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-07-28T19:49:07.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:07 smithi100 bash[82644]: cluster 2024-07-28T19:49:06.070392+0000 mgr.y (mgr.25358) 495 : cluster [DBG] pgmap v1055: 105 pgs: 105 active+clean; 666 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 156 KiB/s rd, 6.5 KiB/s wr, 257 op/s 2024-07-28T19:49:07.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:07 smithi100 bash[82644]: cluster 2024-07-28T19:49:06.333954+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-07-28T19:49:07.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:07 smithi138 bash[67894]: cluster 2024-07-28T19:49:06.070392+0000 mgr.y (mgr.25358) 495 : cluster [DBG] pgmap v1055: 105 pgs: 105 active+clean; 666 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 156 KiB/s rd, 6.5 KiB/s wr, 257 op/s 2024-07-28T19:49:07.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:07 smithi138 bash[67894]: cluster 2024-07-28T19:49:06.333954+0000 mon.a (mon.0) 1515 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-07-28T19:49:08.384 INFO:tasks.workunit.client.0.smithi100.stdout:from 0 to HEAD diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:08.384 INFO:tasks.workunit.client.0.smithi100.stdout: limited diff [0~25165824,29360128~29360128,67108864~46137344,117440512~159383552,281018368~79691776,364904448~50331648] 2024-07-28T19:49:08.401 INFO:tasks.workunit.client.0.smithi100.stdout: actual was [0~25165824,29360128~33554432,67108864~348127232] 2024-07-28T19:49:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:08 smithi100 bash[80199]: cluster 2024-07-28T19:49:07.346809+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-07-28T19:49:08.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:08 smithi100 bash[80199]: audit 2024-07-28T19:49:07.645127+0000 mon.a (mon.0) 1517 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:08.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:08 smithi100 bash[82644]: cluster 2024-07-28T19:49:07.346809+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-07-28T19:49:08.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:08 smithi100 bash[82644]: audit 2024-07-28T19:49:07.645127+0000 mon.a (mon.0) 1517 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:08.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:08 smithi138 bash[67894]: cluster 2024-07-28T19:49:07.346809+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-07-28T19:49:08.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:08 smithi138 bash[67894]: audit 2024-07-28T19:49:07.645127+0000 mon.a (mon.0) 1517 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:09.406 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (39822 ms) 2024-07-28T19:49:09.406 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateRegression6926 2024-07-28T19:49:09.406 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:49:09.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:09 smithi100 bash[80199]: cluster 2024-07-28T19:49:08.071192+0000 mgr.y (mgr.25358) 496 : cluster [DBG] pgmap v1058: 105 pgs: 105 active+clean; 646 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 87 op/s 2024-07-28T19:49:09.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:09 smithi100 bash[80199]: cluster 2024-07-28T19:49:08.360558+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-07-28T19:49:09.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:09 smithi100 bash[82644]: cluster 2024-07-28T19:49:08.071192+0000 mgr.y (mgr.25358) 496 : cluster [DBG] pgmap v1058: 105 pgs: 105 active+clean; 646 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 87 op/s 2024-07-28T19:49:09.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:09 smithi100 bash[82644]: cluster 2024-07-28T19:49:08.360558+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-07-28T19:49:09.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:09 smithi138 bash[67894]: cluster 2024-07-28T19:49:08.071192+0000 mgr.y (mgr.25358) 496 : cluster [DBG] pgmap v1058: 105 pgs: 105 active+clean; 646 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 87 op/s 2024-07-28T19:49:09.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:09 smithi138 bash[67894]: cluster 2024-07-28T19:49:08.360558+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-07-28T19:49:10.328 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:49:10.339 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateRegression6926 (933 ms) 2024-07-28T19:49:10.339 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-07-28T19:49:10.339 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:49:10.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:10 smithi100 bash[80199]: cluster 2024-07-28T19:49:09.357342+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-07-28T19:49:10.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:10 smithi100 bash[80199]: cluster 2024-07-28T19:49:10.292530+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-07-28T19:49:10.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:10 smithi100 bash[82644]: cluster 2024-07-28T19:49:09.357342+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-07-28T19:49:10.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:10 smithi100 bash[82644]: cluster 2024-07-28T19:49:10.292530+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-07-28T19:49:10.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:10 smithi138 bash[67894]: cluster 2024-07-28T19:49:09.357342+0000 mon.a (mon.0) 1519 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-07-28T19:49:10.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:10 smithi138 bash[67894]: cluster 2024-07-28T19:49:10.292530+0000 mon.a (mon.0) 1520 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-07-28T19:49:11.521 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:49:11.521 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 4194304~4194304 2024-07-28T19:49:11.521 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 8388608~4194304 2024-07-28T19:49:11.521 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 12582912~4194304 2024-07-28T19:49:11.521 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 16777216~4194304 2024-07-28T19:49:11.586 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 0~4194304 2024-07-28T19:49:11.586 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 4194304~4194304 2024-07-28T19:49:11.586 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 8388608~4194304 2024-07-28T19:49:11.586 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 16777216~4194304 2024-07-28T19:49:11.616 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1277 ms) 2024-07-28T19:49:11.616 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-07-28T19:49:11.616 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:49:11.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:11 smithi100 bash[82644]: cluster 2024-07-28T19:49:10.072097+0000 mgr.y (mgr.25358) 497 : cluster [DBG] pgmap v1061: 105 pgs: 105 active+clean; 532 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 6.0 KiB/s wr, 119 op/s 2024-07-28T19:49:11.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:11 smithi100 bash[80199]: cluster 2024-07-28T19:49:10.072097+0000 mgr.y (mgr.25358) 497 : cluster [DBG] pgmap v1061: 105 pgs: 105 active+clean; 532 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 6.0 KiB/s wr, 119 op/s 2024-07-28T19:49:11.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:11 smithi138 bash[67894]: cluster 2024-07-28T19:49:10.072097+0000 mgr.y (mgr.25358) 497 : cluster [DBG] pgmap v1061: 105 pgs: 105 active+clean; 532 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 6.0 KiB/s wr, 119 op/s 2024-07-28T19:49:12.669 INFO:tasks.workunit.client.0.smithi100.stdout: wrote [0~8388608,12582912~8388608] to clone 2024-07-28T19:49:12.670 INFO:tasks.workunit.client.0.smithi100.stdout: diff was [0~8388608,12582912~8388608] 2024-07-28T19:49:12.680 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1064 ms) 2024-07-28T19:49:12.680 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-07-28T19:49:12.680 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:49:12.728 INFO:tasks.workunit.client.0.smithi100.stdout: wrote [659043~83001,1088675~7439,2277907~71411,8039287~5150,9645401~52252,10852585~9019,11816529~243,13247029~67296,18883380~40259,20454184~46636] 2024-07-28T19:49:12.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:12 smithi100 bash[80199]: cluster 2024-07-28T19:49:11.378111+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2024-07-28T19:49:12.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:12 smithi100 bash[82644]: cluster 2024-07-28T19:49:11.378111+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2024-07-28T19:49:12.752 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (72 ms) 2024-07-28T19:49:12.752 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-07-28T19:49:12.752 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:49:12.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:12 smithi138 bash[67894]: cluster 2024-07-28T19:49:11.378111+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2024-07-28T19:49:12.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:49:12 smithi138 bash[60615]: ts=2024-07-28T19:49:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:49:13.250 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:49:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:49:12] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2024-07-28T19:49:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:13 smithi100 bash[80199]: cluster 2024-07-28T19:49:12.072780+0000 mgr.y (mgr.25358) 498 : cluster [DBG] pgmap v1064: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 6.2 KiB/s wr, 119 op/s 2024-07-28T19:49:13.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:13 smithi100 bash[80199]: cluster 2024-07-28T19:49:12.395609+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-07-28T19:49:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:13 smithi100 bash[82644]: cluster 2024-07-28T19:49:12.072780+0000 mgr.y (mgr.25358) 498 : cluster [DBG] pgmap v1064: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 6.2 KiB/s wr, 119 op/s 2024-07-28T19:49:13.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:13 smithi100 bash[82644]: cluster 2024-07-28T19:49:12.395609+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-07-28T19:49:13.807 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:49:13 smithi138 bash[59346]: logger=sqlstore.transactions t=2024-07-28T19:49:13.629602495Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-07-28T19:49:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:13 smithi138 bash[67894]: cluster 2024-07-28T19:49:12.072780+0000 mgr.y (mgr.25358) 498 : cluster [DBG] pgmap v1064: 105 pgs: 105 active+clean; 480 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 6.2 KiB/s wr, 119 op/s 2024-07-28T19:49:13.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:13 smithi138 bash[67894]: cluster 2024-07-28T19:49:12.395609+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2024-07-28T19:49:14.635 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1883 ms) 2024-07-28T19:49:14.635 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-07-28T19:49:14.635 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:49:14.732 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 5000005~1234 2024-07-28T19:49:14.741 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (106 ms) 2024-07-28T19:49:14.741 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-07-28T19:49:14.741 INFO:tasks.workunit.client.0.smithi100.stdout:using new format! 2024-07-28T19:49:14.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:14 smithi100 bash[80199]: cluster 2024-07-28T19:49:13.403880+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-07-28T19:49:14.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:14 smithi100 bash[82644]: cluster 2024-07-28T19:49:13.403880+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-07-28T19:49:14.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:14 smithi138 bash[67894]: cluster 2024-07-28T19:49:13.403880+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2024-07-28T19:49:14.879 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 8376263~12345 2024-07-28T19:49:14.879 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 8388608~4194304 2024-07-28T19:49:14.879 INFO:tasks.workunit.client.0.smithi100.stdout:iterate_cb 12582912~54321 2024-07-28T19:49:14.894 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (151 ms) 2024-07-28T19:49:14.894 INFO:tasks.workunit.client.0.smithi100.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-07-28T19:49:14.894 INFO:tasks.workunit.client.0.smithi100.stdout:SKIPPING 2024-07-28T19:49:14.894 INFO:tasks.workunit.client.0.smithi100.stdout:[ OK ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-07-28T19:49:15.299 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] 11 tests from DiffIterateTest/1 (53390 ms total) 2024-07-28T19:49:15.299 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:49:15.299 INFO:tasks.workunit.client.0.smithi100.stdout:[----------] Global test environment tear-down 2024-07-28T19:49:15.299 INFO:tasks.workunit.client.0.smithi100.stdout:[==========] 406 tests from 15 test suites ran. (1071576 ms total) 2024-07-28T19:49:15.299 INFO:tasks.workunit.client.0.smithi100.stdout:[ PASSED ] 406 tests. 2024-07-28T19:49:15.299 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:49:15.300 INFO:tasks.workunit.client.0.smithi100.stdout: YOU HAVE 2 DISABLED TESTS 2024-07-28T19:49:15.300 INFO:tasks.workunit.client.0.smithi100.stdout: 2024-07-28T19:49:15.330 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-07-28T19:49:15.330 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-07-28T19:49:15.384 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-07-28T19:49:15.384 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-07-28T19:49:15.651 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:15 smithi100 bash[82644]: cluster 2024-07-28T19:49:14.073842+0000 mgr.y (mgr.25358) 499 : cluster [DBG] pgmap v1067: 105 pgs: 105 active+clean; 482 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 MiB/s wr, 180 op/s 2024-07-28T19:49:15.651 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:15 smithi100 bash[82644]: cluster 2024-07-28T19:49:14.413203+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-07-28T19:49:15.651 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:15 smithi100 bash[82644]: cluster 2024-07-28T19:49:15.295351+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-07-28T19:49:15.651 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:15 smithi100 bash[80199]: cluster 2024-07-28T19:49:14.073842+0000 mgr.y (mgr.25358) 499 : cluster [DBG] pgmap v1067: 105 pgs: 105 active+clean; 482 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 MiB/s wr, 180 op/s 2024-07-28T19:49:15.651 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:15 smithi100 bash[80199]: cluster 2024-07-28T19:49:14.413203+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-07-28T19:49:15.652 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:15 smithi100 bash[80199]: cluster 2024-07-28T19:49:15.295351+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-07-28T19:49:15.653 DEBUG:teuthology.parallel:result is None 2024-07-28T19:49:15.653 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-07-28T19:49:15.666 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-07-28T19:49:15.666 DEBUG:teuthology.orchestra.run.smithi100:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-07-28T19:49:15.726 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-07-28T19:49:15.726 INFO:teuthology.task.sequential:In sequential, running task print... 2024-07-28T19:49:15.726 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-07-28T19:49:15.726 DEBUG:teuthology.parallel:result is None 2024-07-28T19:49:15.726 INFO:teuthology.run_tasks:Running task print... 2024-07-28T19:49:15.736 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-07-28T19:49:15.736 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-07-28T19:49:15.844 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-07-28T19:49:15.844 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-07-28T19:49:15.844 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph mgr dump --format=json 2024-07-28T19:49:15.845 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:15 smithi138 bash[67894]: cluster 2024-07-28T19:49:14.073842+0000 mgr.y (mgr.25358) 499 : cluster [DBG] pgmap v1067: 105 pgs: 105 active+clean; 482 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.7 MiB/s wr, 180 op/s 2024-07-28T19:49:15.845 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:15 smithi138 bash[67894]: cluster 2024-07-28T19:49:14.413203+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2024-07-28T19:49:15.845 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:15 smithi138 bash[67894]: cluster 2024-07-28T19:49:15.295351+0000 mon.a (mon.0) 1525 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2024-07-28T19:49:17.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:17 smithi100 bash[80199]: cluster 2024-07-28T19:49:16.074293+0000 mgr.y (mgr.25358) 500 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:49:17.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:17 smithi100 bash[82644]: cluster 2024-07-28T19:49:16.074293+0000 mgr.y (mgr.25358) 500 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:49:17.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:17 smithi138 bash[67894]: cluster 2024-07-28T19:49:16.074293+0000 mgr.y (mgr.25358) 500 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:49:19.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:19 smithi100 bash[80199]: cluster 2024-07-28T19:49:18.074908+0000 mgr.y (mgr.25358) 501 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:49:19.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:19 smithi100 bash[82644]: cluster 2024-07-28T19:49:18.074908+0000 mgr.y (mgr.25358) 501 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:49:19.810 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:19 smithi138 bash[67894]: cluster 2024-07-28T19:49:18.074908+0000 mgr.y (mgr.25358) 501 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-07-28T19:49:20.633 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:21.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:21 smithi100 bash[80199]: cluster 2024-07-28T19:49:20.075863+0000 mgr.y (mgr.25358) 502 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:21.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:21 smithi100 bash[82644]: cluster 2024-07-28T19:49:20.075863+0000 mgr.y (mgr.25358) 502 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:21.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:21 smithi138 bash[67894]: cluster 2024-07-28T19:49:20.075863+0000 mgr.y (mgr.25358) 502 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:22.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:49:22 smithi138 bash[60615]: ts=2024-07-28T19:49:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:49:22.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:49:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:49:22] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-07-28T19:49:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:23 smithi100 bash[80199]: cluster 2024-07-28T19:49:22.076599+0000 mgr.y (mgr.25358) 503 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:23.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:23 smithi100 bash[80199]: audit 2024-07-28T19:49:22.645528+0000 mon.a (mon.0) 1526 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:23.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:23 smithi100 bash[82644]: cluster 2024-07-28T19:49:22.076599+0000 mgr.y (mgr.25358) 503 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:23.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:23 smithi100 bash[82644]: audit 2024-07-28T19:49:22.645528+0000 mon.a (mon.0) 1526 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:23 smithi138 bash[67894]: cluster 2024-07-28T19:49:22.076599+0000 mgr.y (mgr.25358) 503 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:23 smithi138 bash[67894]: audit 2024-07-28T19:49:22.645528+0000 mon.a (mon.0) 1526 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:24.850 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:49:25.534 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":39,"flags":0,"active_gid":25358,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6800","nonce":2947416154},{"type":"v1","addr":"172.21.15.100:6801","nonce":2947416154}]},"active_addr":"172.21.15.100:6801/2947416154","active_change":"2024-07-28T19:35:37.500812+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24694,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.100:8443/","prometheus":"http://172.21.15.100:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":303,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":3257556752}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":2951101096}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":3355451322}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.100:0","nonce":1494478504}]}]} 2024-07-28T19:49:25.538 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-07-28T19:49:25.538 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-07-28T19:49:25.539 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd dump --format=json 2024-07-28T19:49:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:25 smithi100 bash[80199]: cluster 2024-07-28T19:49:24.077519+0000 mgr.y (mgr.25358) 504 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:25 smithi100 bash[80199]: audit 2024-07-28T19:49:24.842215+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.100:0/3171346160' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:49:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:25 smithi100 bash[82644]: cluster 2024-07-28T19:49:24.077519+0000 mgr.y (mgr.25358) 504 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:25.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:25 smithi100 bash[82644]: audit 2024-07-28T19:49:24.842215+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.100:0/3171346160' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:49:25.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:25 smithi138 bash[67894]: cluster 2024-07-28T19:49:24.077519+0000 mgr.y (mgr.25358) 504 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:25.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:25 smithi138 bash[67894]: audit 2024-07-28T19:49:24.842215+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.100:0/3171346160' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-07-28T19:49:27.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:27 smithi100 bash[80199]: cluster 2024-07-28T19:49:26.078416+0000 mgr.y (mgr.25358) 505 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:27.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:27 smithi100 bash[82644]: cluster 2024-07-28T19:49:26.078416+0000 mgr.y (mgr.25358) 505 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:27.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:27 smithi138 bash[67894]: cluster 2024-07-28T19:49:26.078416+0000 mgr.y (mgr.25358) 505 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:29 smithi100 bash[80199]: cluster 2024-07-28T19:49:28.079164+0000 mgr.y (mgr.25358) 506 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:29.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:29 smithi100 bash[82644]: cluster 2024-07-28T19:49:28.079164+0000 mgr.y (mgr.25358) 506 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:29.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:29 smithi138 bash[67894]: cluster 2024-07-28T19:49:28.079164+0000 mgr.y (mgr.25358) 506 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:30.322 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:31.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:31 smithi100 bash[80199]: cluster 2024-07-28T19:49:30.080149+0000 mgr.y (mgr.25358) 507 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:31.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:31 smithi100 bash[82644]: cluster 2024-07-28T19:49:30.080149+0000 mgr.y (mgr.25358) 507 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:31.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:31 smithi138 bash[67894]: cluster 2024-07-28T19:49:30.080149+0000 mgr.y (mgr.25358) 507 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:32.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:49:32 smithi138 bash[60615]: ts=2024-07-28T19:49:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:49:33.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:49:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:49:32] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-07-28T19:49:33.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:33 smithi100 bash[80199]: cluster 2024-07-28T19:49:32.080872+0000 mgr.y (mgr.25358) 508 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:33 smithi100 bash[82644]: cluster 2024-07-28T19:49:32.080872+0000 mgr.y (mgr.25358) 508 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:33 smithi138 bash[67894]: cluster 2024-07-28T19:49:32.080872+0000 mgr.y (mgr.25358) 508 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:33.871 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:49:33.871 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":961,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","created":"2024-07-28T19:16:00.842363+0000","modified":"2024-07-28T19:49:15.285059+0000","last_up_change":"2024-07-28T19:42:50.318384+0000","last_in_change":"2024-07-28T19:25:08.436673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":40,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-28T19:21:19.886730+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-28T19:27:14.397490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-07-28T19:29:14.173216+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-07-28T19:29:14.664032+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c4a49140-8022-42e2-bba9-333eb0e88545","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":490,"up_from":506,"up_thru":909,"down_at":491,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":2080394453},{"type":"v1","addr":"172.21.15.100:6803","nonce":2080394453}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":2080394453},{"type":"v1","addr":"172.21.15.100:6805","nonce":2080394453}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2080394453},{"type":"v1","addr":"172.21.15.100:6809","nonce":2080394453}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":2080394453},{"type":"v1","addr":"172.21.15.100:6807","nonce":2080394453}]},"public_addr":"172.21.15.100:6803/2080394453","cluster_addr":"172.21.15.100:6805/2080394453","heartbeat_back_addr":"172.21.15.100:6809/2080394453","heartbeat_front_addr":"172.21.15.100:6807/2080394453","state":["exists","up"]},{"osd":1,"uuid":"4117fbb5-c88e-4d12-bc76-77050b2c0425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":524,"up_from":540,"up_thru":909,"down_at":525,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":1736817357},{"type":"v1","addr":"172.21.15.100:6811","nonce":1736817357}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":1736817357},{"type":"v1","addr":"172.21.15.100:6813","nonce":1736817357}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":1736817357},{"type":"v1","addr":"172.21.15.100:6817","nonce":1736817357}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":1736817357},{"type":"v1","addr":"172.21.15.100:6815","nonce":1736817357}]},"public_addr":"172.21.15.100:6811/1736817357","cluster_addr":"172.21.15.100:6813/1736817357","heartbeat_back_addr":"172.21.15.100:6817/1736817357","heartbeat_front_addr":"172.21.15.100:6815/1736817357","state":["exists","up"]},{"osd":2,"uuid":"f3416396-83c7-42fb-9153-9cb806281347","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":615,"up_from":628,"up_thru":909,"down_at":616,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":286161733},{"type":"v1","addr":"172.21.15.100:6819","nonce":286161733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":286161733},{"type":"v1","addr":"172.21.15.100:6821","nonce":286161733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":286161733},{"type":"v1","addr":"172.21.15.100:6825","nonce":286161733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":286161733},{"type":"v1","addr":"172.21.15.100:6823","nonce":286161733}]},"public_addr":"172.21.15.100:6819/286161733","cluster_addr":"172.21.15.100:6821/286161733","heartbeat_back_addr":"172.21.15.100:6825/286161733","heartbeat_front_addr":"172.21.15.100:6823/286161733","state":["exists","up"]},{"osd":3,"uuid":"0363a205-8262-42a4-aaaf-e52558fec7f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":630,"up_from":643,"up_thru":909,"down_at":631,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":728540967},{"type":"v1","addr":"172.21.15.100:6827","nonce":728540967}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":728540967},{"type":"v1","addr":"172.21.15.100:6829","nonce":728540967}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6832","nonce":728540967},{"type":"v1","addr":"172.21.15.100:6833","nonce":728540967}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":728540967},{"type":"v1","addr":"172.21.15.100:6831","nonce":728540967}]},"public_addr":"172.21.15.100:6827/728540967","cluster_addr":"172.21.15.100:6829/728540967","heartbeat_back_addr":"172.21.15.100:6833/728540967","heartbeat_front_addr":"172.21.15.100:6831/728540967","state":["exists","up"]},{"osd":4,"uuid":"75f51136-e1cc-4d3a-89d8-1618aaff366e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":647,"up_from":652,"up_thru":909,"down_at":647,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":3051177885},{"type":"v1","addr":"172.21.15.138:6801","nonce":3051177885}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":3051177885},{"type":"v1","addr":"172.21.15.138:6803","nonce":3051177885}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":3051177885},{"type":"v1","addr":"172.21.15.138:6807","nonce":3051177885}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":3051177885},{"type":"v1","addr":"172.21.15.138:6805","nonce":3051177885}]},"public_addr":"172.21.15.138:6801/3051177885","cluster_addr":"172.21.15.138:6803/3051177885","heartbeat_back_addr":"172.21.15.138:6807/3051177885","heartbeat_front_addr":"172.21.15.138:6805/3051177885","state":["exists","up"]},{"osd":5,"uuid":"0a3e4950-a201-4105-9e45-ef45ce2d5f3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":909,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6809","nonce":1102784487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6811","nonce":1102784487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6815","nonce":1102784487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6813","nonce":1102784487}]},"public_addr":"172.21.15.138:6809/1102784487","cluster_addr":"172.21.15.138:6811/1102784487","heartbeat_back_addr":"172.21.15.138:6815/1102784487","heartbeat_front_addr":"172.21.15.138:6813/1102784487","state":["exists","up"]},{"osd":6,"uuid":"8a20da48-aee9-44d8-a3db-63f916c6b763","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":909,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6817","nonce":3709444988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6819","nonce":3709444988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6823","nonce":3709444988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6821","nonce":3709444988}]},"public_addr":"172.21.15.138:6817/3709444988","cluster_addr":"172.21.15.138:6819/3709444988","heartbeat_back_addr":"172.21.15.138:6823/3709444988","heartbeat_front_addr":"172.21.15.138:6821/3709444988","state":["exists","up"]},{"osd":7,"uuid":"d52b6b70-462f-42ed-966f-fa14921ba172","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":909,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6825","nonce":1158882219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6827","nonce":1158882219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6831","nonce":1158882219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6829","nonce":1158882219}]},"public_addr":"172.21.15.138:6825/1158882219","cluster_addr":"172.21.15.138:6827/1158882219","heartbeat_back_addr":"172.21.15.138:6831/1158882219","heartbeat_front_addr":"172.21.15.138:6829/1158882219","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:19:30.916069+0000","dead_epoch":490},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:20:23.352940+0000","dead_epoch":524},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:21:16.853645+0000","dead_epoch":614},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:09.540873+0000","dead_epoch":630},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:59.198788+0000","dead_epoch":647},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:23:49.135324+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:24:40.795346+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:25:32.389079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.100:0/1274339179":"2024-07-28T20:45:19.437632+0000","172.21.15.100:0/3094956043":"2024-07-28T20:40:13.071156+0000","172.21.15.100:0/3253252073":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/376427751":"2024-07-29T19:33:32.301959+0000","172.21.15.138:0/3916218581":"2024-07-29T19:34:34.954081+0000","172.21.15.100:6801/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6800/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6835/4040051651":"2024-07-29T19:29:22.449384+0000","172.21.15.100:6834/4040051651":"2024-07-29T19:29:22.449384+0000","172.21.15.100:0/4293227232":"2024-07-28T20:40:12.080911+0000","172.21.15.100:6801/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.138:0/4175092009":"2024-07-29T19:34:34.954081+0000","172.21.15.100:6800/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.138:6835/143155515":"2024-07-29T19:34:34.954081+0000","172.21.15.138:0/3708581135":"2024-07-29T19:34:34.954081+0000","172.21.15.138:6834/143155515":"2024-07-29T19:34:34.954081+0000","172.21.15.100:0/3944711986":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6800/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2046048210":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/2291421462":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/989116380":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/1006451977":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/3154304581":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/480614525":"2024-07-29T19:33:32.301959+0000","172.21.15.100:6801/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3009795473":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/3916188162":"2024-07-29T19:33:32.301959+0000","172.21.15.100:0/3784806487":"2024-07-29T19:33:32.301959+0000","172.21.15.100:6800/1276184411":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/3152621533":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/1970641336":"2024-07-29T19:35:37.499776+0000","172.21.15.100:6801/1276184411":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/553075364":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/557847650":"2024-07-29T19:35:37.499776+0000","172.21.15.138:0/1447892926":"2024-07-29T19:34:34.954081+0000","172.21.15.100:0/638886582":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/2276448354":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/2863303498":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/3841450082":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6800/2345539946":"2024-07-29T19:33:32.301959+0000","172.21.15.100:6801/2345539946":"2024-07-29T19:33:32.301959+0000","172.21.15.138:0/1136533310":"2024-07-29T19:34:34.954081+0000","172.21.15.138:0/3705378738":"2024-07-29T19:34:34.954081+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":231,"length":1}]},{"pool":13,"snaps":[{"begin":14,"length":1}]},{"pool":15,"snaps":[{"begin":62,"length":2},{"begin":70,"length":1},{"begin":73,"length":1}]},{"pool":16,"snaps":[{"begin":31,"length":1}]},{"pool":17,"snaps":[{"begin":2,"length":1}]},{"pool":39,"snaps":[{"begin":9,"length":2}]},{"pool":40,"snaps":[{"begin":9,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":40,"snaps":[{"begin":10,"length":5}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-28T19:49:34.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:34 smithi100 bash[80199]: audit 2024-07-28T19:49:33.871286+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.100:0/542006647' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:49:34.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:34 smithi100 bash[82644]: audit 2024-07-28T19:49:33.871286+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.100:0/542006647' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:49:34.774 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-07-28T19:49:34.775 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd dump --format=json 2024-07-28T19:49:34.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:34 smithi138 bash[67894]: audit 2024-07-28T19:49:33.871286+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.100:0/542006647' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:49:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:35 smithi100 bash[80199]: cluster 2024-07-28T19:49:34.081762+0000 mgr.y (mgr.25358) 509 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:35.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:35 smithi100 bash[82644]: cluster 2024-07-28T19:49:34.081762+0000 mgr.y (mgr.25358) 509 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:35 smithi138 bash[67894]: cluster 2024-07-28T19:49:34.081762+0000 mgr.y (mgr.25358) 509 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:37.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:37 smithi100 bash[80199]: cluster 2024-07-28T19:49:36.082601+0000 mgr.y (mgr.25358) 510 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:37.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:37 smithi100 bash[82644]: cluster 2024-07-28T19:49:36.082601+0000 mgr.y (mgr.25358) 510 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:37.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:37 smithi138 bash[67894]: cluster 2024-07-28T19:49:36.082601+0000 mgr.y (mgr.25358) 510 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:38.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:38 smithi100 bash[80199]: audit 2024-07-28T19:49:37.646158+0000 mon.a (mon.0) 1527 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:38.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:38 smithi100 bash[82644]: audit 2024-07-28T19:49:37.646158+0000 mon.a (mon.0) 1527 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:38.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:38 smithi138 bash[67894]: audit 2024-07-28T19:49:37.646158+0000 mon.a (mon.0) 1527 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:39.554 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:39.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:39 smithi100 bash[80199]: cluster 2024-07-28T19:49:38.083382+0000 mgr.y (mgr.25358) 511 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:39.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:39 smithi100 bash[82644]: cluster 2024-07-28T19:49:38.083382+0000 mgr.y (mgr.25358) 511 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:39.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:39 smithi138 bash[67894]: cluster 2024-07-28T19:49:38.083382+0000 mgr.y (mgr.25358) 511 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:40.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:40 smithi100 bash[80199]: cluster 2024-07-28T19:49:40.084357+0000 mgr.y (mgr.25358) 512 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:40.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:40 smithi100 bash[82644]: cluster 2024-07-28T19:49:40.084357+0000 mgr.y (mgr.25358) 512 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:40.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:40 smithi138 bash[67894]: cluster 2024-07-28T19:49:40.084357+0000 mgr.y (mgr.25358) 512 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:42.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:49:42 smithi138 bash[60615]: ts=2024-07-28T19:49:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:49:43.092 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:49:43.092 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":961,"fsid":"9170a172-4d15-11ef-bcc8-c7b262605968","created":"2024-07-28T19:16:00.842363+0000","modified":"2024-07-28T19:49:15.285059+0000","last_up_change":"2024-07-28T19:42:50.318384+0000","last_in_change":"2024-07-28T19:25:08.436673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":40,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-07-28T19:21:19.886730+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-07-28T19:27:14.397490+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-07-28T19:29:14.173216+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-07-28T19:29:14.664032+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c4a49140-8022-42e2-bba9-333eb0e88545","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":490,"up_from":506,"up_thru":909,"down_at":491,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":2080394453},{"type":"v1","addr":"172.21.15.100:6803","nonce":2080394453}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":2080394453},{"type":"v1","addr":"172.21.15.100:6805","nonce":2080394453}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2080394453},{"type":"v1","addr":"172.21.15.100:6809","nonce":2080394453}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":2080394453},{"type":"v1","addr":"172.21.15.100:6807","nonce":2080394453}]},"public_addr":"172.21.15.100:6803/2080394453","cluster_addr":"172.21.15.100:6805/2080394453","heartbeat_back_addr":"172.21.15.100:6809/2080394453","heartbeat_front_addr":"172.21.15.100:6807/2080394453","state":["exists","up"]},{"osd":1,"uuid":"4117fbb5-c88e-4d12-bc76-77050b2c0425","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":524,"up_from":540,"up_thru":909,"down_at":525,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":1736817357},{"type":"v1","addr":"172.21.15.100:6811","nonce":1736817357}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":1736817357},{"type":"v1","addr":"172.21.15.100:6813","nonce":1736817357}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":1736817357},{"type":"v1","addr":"172.21.15.100:6817","nonce":1736817357}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":1736817357},{"type":"v1","addr":"172.21.15.100:6815","nonce":1736817357}]},"public_addr":"172.21.15.100:6811/1736817357","cluster_addr":"172.21.15.100:6813/1736817357","heartbeat_back_addr":"172.21.15.100:6817/1736817357","heartbeat_front_addr":"172.21.15.100:6815/1736817357","state":["exists","up"]},{"osd":2,"uuid":"f3416396-83c7-42fb-9153-9cb806281347","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":615,"up_from":628,"up_thru":909,"down_at":616,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":286161733},{"type":"v1","addr":"172.21.15.100:6819","nonce":286161733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":286161733},{"type":"v1","addr":"172.21.15.100:6821","nonce":286161733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":286161733},{"type":"v1","addr":"172.21.15.100:6825","nonce":286161733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":286161733},{"type":"v1","addr":"172.21.15.100:6823","nonce":286161733}]},"public_addr":"172.21.15.100:6819/286161733","cluster_addr":"172.21.15.100:6821/286161733","heartbeat_back_addr":"172.21.15.100:6825/286161733","heartbeat_front_addr":"172.21.15.100:6823/286161733","state":["exists","up"]},{"osd":3,"uuid":"0363a205-8262-42a4-aaaf-e52558fec7f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":630,"up_from":643,"up_thru":909,"down_at":631,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":728540967},{"type":"v1","addr":"172.21.15.100:6827","nonce":728540967}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":728540967},{"type":"v1","addr":"172.21.15.100:6829","nonce":728540967}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6832","nonce":728540967},{"type":"v1","addr":"172.21.15.100:6833","nonce":728540967}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":728540967},{"type":"v1","addr":"172.21.15.100:6831","nonce":728540967}]},"public_addr":"172.21.15.100:6827/728540967","cluster_addr":"172.21.15.100:6829/728540967","heartbeat_back_addr":"172.21.15.100:6833/728540967","heartbeat_front_addr":"172.21.15.100:6831/728540967","state":["exists","up"]},{"osd":4,"uuid":"75f51136-e1cc-4d3a-89d8-1618aaff366e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":647,"up_from":652,"up_thru":909,"down_at":647,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":3051177885},{"type":"v1","addr":"172.21.15.138:6801","nonce":3051177885}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":3051177885},{"type":"v1","addr":"172.21.15.138:6803","nonce":3051177885}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":3051177885},{"type":"v1","addr":"172.21.15.138:6807","nonce":3051177885}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":3051177885},{"type":"v1","addr":"172.21.15.138:6805","nonce":3051177885}]},"public_addr":"172.21.15.138:6801/3051177885","cluster_addr":"172.21.15.138:6803/3051177885","heartbeat_back_addr":"172.21.15.138:6807/3051177885","heartbeat_front_addr":"172.21.15.138:6805/3051177885","state":["exists","up"]},{"osd":5,"uuid":"0a3e4950-a201-4105-9e45-ef45ce2d5f3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":909,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6809","nonce":1102784487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6811","nonce":1102784487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6815","nonce":1102784487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":1102784487},{"type":"v1","addr":"172.21.15.138:6813","nonce":1102784487}]},"public_addr":"172.21.15.138:6809/1102784487","cluster_addr":"172.21.15.138:6811/1102784487","heartbeat_back_addr":"172.21.15.138:6815/1102784487","heartbeat_front_addr":"172.21.15.138:6813/1102784487","state":["exists","up"]},{"osd":6,"uuid":"8a20da48-aee9-44d8-a3db-63f916c6b763","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":909,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6817","nonce":3709444988}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6819","nonce":3709444988}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6823","nonce":3709444988}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3709444988},{"type":"v1","addr":"172.21.15.138:6821","nonce":3709444988}]},"public_addr":"172.21.15.138:6817/3709444988","cluster_addr":"172.21.15.138:6819/3709444988","heartbeat_back_addr":"172.21.15.138:6823/3709444988","heartbeat_front_addr":"172.21.15.138:6821/3709444988","state":["exists","up"]},{"osd":7,"uuid":"d52b6b70-462f-42ed-966f-fa14921ba172","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":909,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6825","nonce":1158882219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6827","nonce":1158882219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6831","nonce":1158882219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1158882219},{"type":"v1","addr":"172.21.15.138:6829","nonce":1158882219}]},"public_addr":"172.21.15.138:6825/1158882219","cluster_addr":"172.21.15.138:6827/1158882219","heartbeat_back_addr":"172.21.15.138:6831/1158882219","heartbeat_front_addr":"172.21.15.138:6829/1158882219","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:19:30.916069+0000","dead_epoch":490},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:20:23.352940+0000","dead_epoch":524},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:21:16.853645+0000","dead_epoch":614},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:09.540873+0000","dead_epoch":630},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:22:59.198788+0000","dead_epoch":647},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:23:49.135324+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:24:40.795346+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-07-28T19:25:32.389079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.100:0/1274339179":"2024-07-28T20:45:19.437632+0000","172.21.15.100:0/3094956043":"2024-07-28T20:40:13.071156+0000","172.21.15.100:0/3253252073":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/557847650":"2024-07-29T19:35:37.499776+0000","172.21.15.100:6800/1276184411":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/376427751":"2024-07-29T19:33:32.301959+0000","172.21.15.100:0/1970641336":"2024-07-29T19:35:37.499776+0000","172.21.15.100:6801/1276184411":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/3152621533":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/4293227232":"2024-07-28T20:40:12.080911+0000","172.21.15.100:6801/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.138:0/4175092009":"2024-07-29T19:34:34.954081+0000","172.21.15.100:6800/728100671":"2024-07-29T19:26:30.807202+0000","172.21.15.138:6835/143155515":"2024-07-29T19:34:34.954081+0000","172.21.15.138:0/3708581135":"2024-07-29T19:34:34.954081+0000","172.21.15.138:6834/143155515":"2024-07-29T19:34:34.954081+0000","172.21.15.100:0/3944711986":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6800/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2046048210":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/2291421462":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/989116380":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/1006451977":"2024-07-29T19:26:30.807202+0000","172.21.15.100:6801/3959196626":"2024-07-29T19:16:28.286850+0000","172.21.15.138:0/3916218581":"2024-07-29T19:34:34.954081+0000","172.21.15.100:0/3154304581":"2024-07-29T19:26:30.807202+0000","172.21.15.100:0/480614525":"2024-07-29T19:33:32.301959+0000","172.21.15.100:6801/2028766495":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/3009795473":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/3916188162":"2024-07-29T19:33:32.301959+0000","172.21.15.100:0/3784806487":"2024-07-29T19:33:32.301959+0000","172.21.15.100:0/2863303498":"2024-07-29T19:16:28.286850+0000","172.21.15.100:0/3841450082":"2024-07-29T19:16:28.286850+0000","172.21.15.100:6800/2345539946":"2024-07-29T19:33:32.301959+0000","172.21.15.100:6801/2345539946":"2024-07-29T19:33:32.301959+0000","172.21.15.100:0/553075364":"2024-07-29T19:16:59.283147+0000","172.21.15.100:0/2276448354":"2024-07-29T19:35:37.499776+0000","172.21.15.100:0/638886582":"2024-07-29T19:26:30.807202+0000","172.21.15.138:0/1447892926":"2024-07-29T19:34:34.954081+0000","172.21.15.100:6834/4040051651":"2024-07-29T19:29:22.449384+0000","172.21.15.138:0/1136533310":"2024-07-29T19:34:34.954081+0000","172.21.15.138:0/3705378738":"2024-07-29T19:34:34.954081+0000","172.21.15.100:6835/4040051651":"2024-07-29T19:29:22.449384+0000","172.21.15.100:6800/3959196626":"2024-07-29T19:16:28.286850+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":231,"length":1}]},{"pool":13,"snaps":[{"begin":14,"length":1}]},{"pool":15,"snaps":[{"begin":62,"length":2},{"begin":70,"length":1},{"begin":73,"length":1}]},{"pool":16,"snaps":[{"begin":31,"length":1}]},{"pool":17,"snaps":[{"begin":2,"length":1}]},{"pool":39,"snaps":[{"begin":9,"length":2}]},{"pool":40,"snaps":[{"begin":9,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":40,"snaps":[{"begin":10,"length":5}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-07-28T19:49:43.104 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:49:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:49:42] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-07-28T19:49:43.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:43 smithi100 bash[82644]: cluster 2024-07-28T19:49:42.085113+0000 mgr.y (mgr.25358) 513 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:43.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:43 smithi100 bash[82644]: audit 2024-07-28T19:49:43.090572+0000 mon.a (mon.0) 1528 : audit [DBG] from='client.? 172.21.15.100:0/2841162422' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:49:43.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:43 smithi100 bash[80199]: cluster 2024-07-28T19:49:42.085113+0000 mgr.y (mgr.25358) 513 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:43.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:43 smithi100 bash[80199]: audit 2024-07-28T19:49:43.090572+0000 mon.a (mon.0) 1528 : audit [DBG] from='client.? 172.21.15.100:0/2841162422' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:49:43.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:43 smithi138 bash[67894]: cluster 2024-07-28T19:49:42.085113+0000 mgr.y (mgr.25358) 513 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:43.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:43 smithi138 bash[67894]: audit 2024-07-28T19:49:43.090572+0000 mon.a (mon.0) 1528 : audit [DBG] from='client.? 172.21.15.100:0/2841162422' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-07-28T19:49:43.842 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-07-28T19:49:43.843 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-07-28T19:49:43.843 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-07-28T19:49:43.844 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-07-28T19:49:43.844 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-07-28T19:49:43.844 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-07-28T19:49:43.845 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-07-28T19:49:43.845 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-07-28T19:49:44.153 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:44 smithi100 bash[80199]: audit 2024-07-28T19:49:44.040885+0000 mon.a (mon.0) 1529 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:49:44.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:44 smithi100 bash[82644]: audit 2024-07-28T19:49:44.040885+0000 mon.a (mon.0) 1529 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:49:44.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:44 smithi138 bash[67894]: audit 2024-07-28T19:49:44.040885+0000 mon.a (mon.0) 1529 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:49:45.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:45 smithi100 bash[80199]: cluster 2024-07-28T19:49:44.085741+0000 mgr.y (mgr.25358) 514 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:45.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:45 smithi100 bash[82644]: cluster 2024-07-28T19:49:44.085741+0000 mgr.y (mgr.25358) 514 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:45.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:45 smithi138 bash[67894]: cluster 2024-07-28T19:49:44.085741+0000 mgr.y (mgr.25358) 514 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:47.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:47 smithi100 bash[80199]: cluster 2024-07-28T19:49:46.086603+0000 mgr.y (mgr.25358) 515 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:47.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:47 smithi100 bash[82644]: cluster 2024-07-28T19:49:46.086603+0000 mgr.y (mgr.25358) 515 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:47.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:47 smithi138 bash[67894]: cluster 2024-07-28T19:49:46.086603+0000 mgr.y (mgr.25358) 515 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:48.766 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:48.766 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:48.767 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:48.768 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:48.768 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:48.773 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:48.773 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:48.773 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:49:49.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:49 smithi100 bash[80199]: cluster 2024-07-28T19:49:48.087420+0000 mgr.y (mgr.25358) 516 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:49.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:49 smithi100 bash[82644]: cluster 2024-07-28T19:49:48.087420+0000 mgr.y (mgr.25358) 516 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:49.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:49 smithi138 bash[67894]: cluster 2024-07-28T19:49:48.087420+0000 mgr.y (mgr.25358) 516 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:50.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[80199]: audit 2024-07-28T19:49:49.488968+0000 mon.a (mon.0) 1530 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[80199]: audit 2024-07-28T19:49:49.498641+0000 mon.a (mon.0) 1531 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[80199]: audit 2024-07-28T19:49:49.899026+0000 mon.a (mon.0) 1532 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[80199]: audit 2024-07-28T19:49:49.907966+0000 mon.a (mon.0) 1533 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[80199]: cluster 2024-07-28T19:49:50.088413+0000 mgr.y (mgr.25358) 517 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[80199]: audit 2024-07-28T19:49:50.362716+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:49:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[80199]: audit 2024-07-28T19:49:50.363824+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:49:50.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[80199]: audit 2024-07-28T19:49:50.374265+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[82644]: audit 2024-07-28T19:49:49.488968+0000 mon.a (mon.0) 1530 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[82644]: audit 2024-07-28T19:49:49.498641+0000 mon.a (mon.0) 1531 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[82644]: audit 2024-07-28T19:49:49.899026+0000 mon.a (mon.0) 1532 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[82644]: audit 2024-07-28T19:49:49.907966+0000 mon.a (mon.0) 1533 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[82644]: cluster 2024-07-28T19:49:50.088413+0000 mgr.y (mgr.25358) 517 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[82644]: audit 2024-07-28T19:49:50.362716+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:49:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[82644]: audit 2024-07-28T19:49:50.363824+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:49:50.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:50 smithi100 bash[82644]: audit 2024-07-28T19:49:50.374265+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:50 smithi138 bash[67894]: audit 2024-07-28T19:49:49.488968+0000 mon.a (mon.0) 1530 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:50 smithi138 bash[67894]: audit 2024-07-28T19:49:49.498641+0000 mon.a (mon.0) 1531 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:50 smithi138 bash[67894]: audit 2024-07-28T19:49:49.899026+0000 mon.a (mon.0) 1532 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:50 smithi138 bash[67894]: audit 2024-07-28T19:49:49.907966+0000 mon.a (mon.0) 1533 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:50.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:50 smithi138 bash[67894]: cluster 2024-07-28T19:49:50.088413+0000 mgr.y (mgr.25358) 517 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:50.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:50 smithi138 bash[67894]: audit 2024-07-28T19:49:50.362716+0000 mon.a (mon.0) 1534 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:49:50.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:50 smithi138 bash[67894]: audit 2024-07-28T19:49:50.363824+0000 mon.a (mon.0) 1535 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:49:50.808 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:50 smithi138 bash[67894]: audit 2024-07-28T19:49:50.374265+0000 mon.a (mon.0) 1536 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:49:52.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:49:52 smithi138 bash[60615]: ts=2024-07-28T19:49:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:49:52.998 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:49:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:49:52] "GET /metrics HTTP/1.1" 200 34260 "" "Prometheus/2.43.0" 2024-07-28T19:49:53.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:53 smithi100 bash[80199]: cluster 2024-07-28T19:49:52.088966+0000 mgr.y (mgr.25358) 518 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:53.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:53 smithi100 bash[80199]: audit 2024-07-28T19:49:52.647250+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:53.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:53 smithi100 bash[82644]: cluster 2024-07-28T19:49:52.088966+0000 mgr.y (mgr.25358) 518 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:53.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:53 smithi100 bash[82644]: audit 2024-07-28T19:49:52.647250+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:53.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:53 smithi138 bash[67894]: cluster 2024-07-28T19:49:52.088966+0000 mgr.y (mgr.25358) 518 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:53.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:53 smithi138 bash[67894]: audit 2024-07-28T19:49:52.647250+0000 mon.a (mon.0) 1537 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:49:55.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:55 smithi100 bash[82644]: cluster 2024-07-28T19:49:54.089805+0000 mgr.y (mgr.25358) 519 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:55.249 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:55 smithi100 bash[80199]: cluster 2024-07-28T19:49:54.089805+0000 mgr.y (mgr.25358) 519 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:55.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:55 smithi138 bash[67894]: cluster 2024-07-28T19:49:54.089805+0000 mgr.y (mgr.25358) 519 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:57.491 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:57 smithi100 bash[80199]: cluster 2024-07-28T19:49:56.090711+0000 mgr.y (mgr.25358) 520 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:57.492 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:57 smithi100 bash[82644]: cluster 2024-07-28T19:49:56.090711+0000 mgr.y (mgr.25358) 520 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:57.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:57 smithi138 bash[67894]: cluster 2024-07-28T19:49:56.090711+0000 mgr.y (mgr.25358) 520 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:49:59 smithi100 bash[80199]: cluster 2024-07-28T19:49:58.091470+0000 mgr.y (mgr.25358) 521 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:59.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:49:59 smithi100 bash[82644]: cluster 2024-07-28T19:49:58.091470+0000 mgr.y (mgr.25358) 521 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:49:59.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:49:59 smithi138 bash[67894]: cluster 2024-07-28T19:49:58.091470+0000 mgr.y (mgr.25358) 521 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:00.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:50:00.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:50:00.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.345 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: debug 2024-07-28T19:49:59.994+0000 7f8fc6d0c640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000137+0000 mon.a (mon.0) 1538 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000177+0000 mon.a (mon.0) 1539 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000194+0000 mon.a (mon.0) 1540 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000217+0000 mon.a (mon.0) 1541 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000237+0000 mon.a (mon.0) 1542 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000278+0000 mon.a (mon.0) 1543 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000311+0000 mon.a (mon.0) 1544 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.346 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000339+0000 mon.a (mon.0) 1545 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000360+0000 mon.a (mon.0) 1546 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.000378+0000 mon.a (mon.0) 1547 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000137+0000 mon.a (mon.0) 1538 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000177+0000 mon.a (mon.0) 1539 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000194+0000 mon.a (mon.0) 1540 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000217+0000 mon.a (mon.0) 1541 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000237+0000 mon.a (mon.0) 1542 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000278+0000 mon.a (mon.0) 1543 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000311+0000 mon.a (mon.0) 1544 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000339+0000 mon.a (mon.0) 1545 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.347 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000360+0000 mon.a (mon.0) 1546 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.348 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:00 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.000378+0000 mon.a (mon.0) 1547 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000137+0000 mon.a (mon.0) 1538 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-07-28T19:50:00.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000177+0000 mon.a (mon.0) 1539 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-07-28T19:50:00.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000194+0000 mon.a (mon.0) 1540 : cluster [ERR] osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000217+0000 mon.a (mon.0) 1541 : cluster [ERR] osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000237+0000 mon.a (mon.0) 1542 : cluster [ERR] osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000278+0000 mon.a (mon.0) 1543 : cluster [ERR] osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000311+0000 mon.a (mon.0) 1544 : cluster [ERR] osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000339+0000 mon.a (mon.0) 1545 : cluster [ERR] osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000360+0000 mon.a (mon.0) 1546 : cluster [ERR] osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:00.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:00 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.000378+0000 mon.a (mon.0) 1547 : cluster [ERR] osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.2/32,172.21.0.0/20' subnet 2024-07-28T19:50:01.488 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:01 smithi100 bash[80199]: cluster 2024-07-28T19:50:00.092439+0000 mgr.y (mgr.25358) 522 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:01.488 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:01 smithi100 bash[82644]: cluster 2024-07-28T19:50:00.092439+0000 mgr.y (mgr.25358) 522 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:01.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:01 smithi138 bash[67894]: cluster 2024-07-28T19:50:00.092439+0000 mgr.y (mgr.25358) 522 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:02.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:50:02 smithi138 bash[60615]: ts=2024-07-28T19:50:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:50:03.125 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:50:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:50:02] "GET /metrics HTTP/1.1" 200 34260 "" "Prometheus/2.43.0" 2024-07-28T19:50:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:03 smithi100 bash[82644]: cluster 2024-07-28T19:50:02.093315+0000 mgr.y (mgr.25358) 523 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:03.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:03 smithi100 bash[80199]: cluster 2024-07-28T19:50:02.093315+0000 mgr.y (mgr.25358) 523 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:03 smithi138 bash[67894]: cluster 2024-07-28T19:50:02.093315+0000 mgr.y (mgr.25358) 523 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:03.768 INFO:teuthology.orchestra.run.smithi100.stdout:2761663971422 2024-07-28T19:50:03.768 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-07-28T19:50:05.297 INFO:teuthology.orchestra.run.smithi100.stdout:2319282339959 2024-07-28T19:50:05.297 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-07-28T19:50:05.429 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:05 smithi100 bash[80199]: cluster 2024-07-28T19:50:04.094285+0000 mgr.y (mgr.25358) 524 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:05.430 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:05 smithi100 bash[82644]: cluster 2024-07-28T19:50:04.094285+0000 mgr.y (mgr.25358) 524 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:05.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:05 smithi138 bash[67894]: cluster 2024-07-28T19:50:04.094285+0000 mgr.y (mgr.25358) 524 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:06.876 INFO:teuthology.orchestra.run.smithi100.stdout:167503724863 2024-07-28T19:50:06.876 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-07-28T19:50:07.457 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:07 smithi100 bash[80199]: cluster 2024-07-28T19:50:06.095261+0000 mgr.y (mgr.25358) 525 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:07.458 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:07 smithi100 bash[82644]: cluster 2024-07-28T19:50:06.095261+0000 mgr.y (mgr.25358) 525 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:07.477 INFO:teuthology.orchestra.run.smithi100.stdout:236223201579 2024-07-28T19:50:07.477 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-07-28T19:50:07.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:07 smithi138 bash[67894]: cluster 2024-07-28T19:50:06.095261+0000 mgr.y (mgr.25358) 525 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:08.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:08 smithi100 bash[80199]: audit 2024-07-28T19:50:07.646119+0000 mon.a (mon.0) 1548 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:08.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:08 smithi100 bash[82644]: audit 2024-07-28T19:50:07.646119+0000 mon.a (mon.0) 1548 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:08.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:08 smithi138 bash[67894]: audit 2024-07-28T19:50:07.646119+0000 mon.a (mon.0) 1548 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:09.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:09 smithi138 bash[67894]: cluster 2024-07-28T19:50:08.096164+0000 mgr.y (mgr.25358) 526 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:09.719 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:09 smithi100 bash[80199]: cluster 2024-07-28T19:50:08.096164+0000 mgr.y (mgr.25358) 526 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:09.720 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:09 smithi100 bash[82644]: cluster 2024-07-28T19:50:08.096164+0000 mgr.y (mgr.25358) 526 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:09.721 INFO:teuthology.orchestra.run.smithi100.stdout:2697239461989 2024-07-28T19:50:09.721 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-07-28T19:50:09.732 INFO:teuthology.orchestra.run.smithi100.stdout:2173253451903 2024-07-28T19:50:09.732 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-07-28T19:50:09.738 INFO:teuthology.orchestra.run.smithi100.stdout:2800318677081 2024-07-28T19:50:09.739 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-07-28T19:50:09.796 INFO:teuthology.orchestra.run.smithi100.stdout:201863463221 2024-07-28T19:50:09.797 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-07-28T19:50:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:11 smithi100 bash[80199]: cluster 2024-07-28T19:50:10.097193+0000 mgr.y (mgr.25358) 527 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:11.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:11 smithi100 bash[82644]: cluster 2024-07-28T19:50:10.097193+0000 mgr.y (mgr.25358) 527 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:11 smithi138 bash[67894]: cluster 2024-07-28T19:50:10.097193+0000 mgr.y (mgr.25358) 527 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:12.618 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:12.618 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:12.619 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:12.619 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:12.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:50:12 smithi138 bash[60615]: ts=2024-07-28T19:50:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:50:13.193 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:50:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:50:12] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-07-28T19:50:13.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:13 smithi100 bash[80199]: cluster 2024-07-28T19:50:12.097979+0000 mgr.y (mgr.25358) 528 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:13 smithi100 bash[82644]: cluster 2024-07-28T19:50:12.097979+0000 mgr.y (mgr.25358) 528 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:13.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:13 smithi138 bash[67894]: cluster 2024-07-28T19:50:12.097979+0000 mgr.y (mgr.25358) 528 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:13.640 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:13.640 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:13.641 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:13.643 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:15.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:15 smithi100 bash[80199]: cluster 2024-07-28T19:50:14.098918+0000 mgr.y (mgr.25358) 529 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:15.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:15 smithi100 bash[82644]: cluster 2024-07-28T19:50:14.098918+0000 mgr.y (mgr.25358) 529 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:15.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:15 smithi138 bash[67894]: cluster 2024-07-28T19:50:14.098918+0000 mgr.y (mgr.25358) 529 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:17.538 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:17 smithi100 bash[80199]: cluster 2024-07-28T19:50:16.099800+0000 mgr.y (mgr.25358) 530 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:17.538 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:17 smithi100 bash[82644]: cluster 2024-07-28T19:50:16.099800+0000 mgr.y (mgr.25358) 530 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:17.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:17 smithi138 bash[67894]: cluster 2024-07-28T19:50:16.099800+0000 mgr.y (mgr.25358) 530 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:19.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:19 smithi138 bash[67894]: cluster 2024-07-28T19:50:18.100544+0000 mgr.y (mgr.25358) 531 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:19.605 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:19 smithi100 bash[80199]: cluster 2024-07-28T19:50:18.100544+0000 mgr.y (mgr.25358) 531 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:19.605 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:19 smithi100 bash[82644]: cluster 2024-07-28T19:50:18.100544+0000 mgr.y (mgr.25358) 531 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:21.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:21 smithi100 bash[80199]: cluster 2024-07-28T19:50:20.101640+0000 mgr.y (mgr.25358) 532 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:21.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:21 smithi100 bash[82644]: cluster 2024-07-28T19:50:20.101640+0000 mgr.y (mgr.25358) 532 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:21.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:21 smithi138 bash[67894]: cluster 2024-07-28T19:50:20.101640+0000 mgr.y (mgr.25358) 532 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:22.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:50:22 smithi138 bash[60615]: ts=2024-07-28T19:50:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:50:23.230 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:50:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:50:22] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-07-28T19:50:23.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:23 smithi100 bash[80199]: cluster 2024-07-28T19:50:22.102407+0000 mgr.y (mgr.25358) 533 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:23.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:23 smithi100 bash[80199]: audit 2024-07-28T19:50:22.646909+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:23.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:23 smithi100 bash[82644]: cluster 2024-07-28T19:50:22.102407+0000 mgr.y (mgr.25358) 533 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:23.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:23 smithi100 bash[82644]: audit 2024-07-28T19:50:22.646909+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:23.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:23 smithi138 bash[67894]: cluster 2024-07-28T19:50:22.102407+0000 mgr.y (mgr.25358) 533 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:23.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:23 smithi138 bash[67894]: audit 2024-07-28T19:50:22.646909+0000 mon.a (mon.0) 1549 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:24.299 INFO:teuthology.orchestra.run.smithi100.stdout:236223201583 2024-07-28T19:50:25.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:25 smithi100 bash[80199]: cluster 2024-07-28T19:50:24.102909+0000 mgr.y (mgr.25358) 534 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:25.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:25 smithi100 bash[80199]: audit 2024-07-28T19:50:24.301110+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.100:0/2989102155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:50:25.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:25 smithi100 bash[82644]: cluster 2024-07-28T19:50:24.102909+0000 mgr.y (mgr.25358) 534 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:25.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:25 smithi100 bash[82644]: audit 2024-07-28T19:50:24.301110+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.100:0/2989102155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:50:25.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:25 smithi138 bash[67894]: cluster 2024-07-28T19:50:24.102909+0000 mgr.y (mgr.25358) 534 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:25.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:25 smithi138 bash[67894]: audit 2024-07-28T19:50:24.301110+0000 mon.a (mon.0) 1550 : audit [DBG] from='client.? 172.21.15.100:0/2989102155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-07-28T19:50:25.729 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201579 got 236223201583 for osd.7 2024-07-28T19:50:25.729 DEBUG:teuthology.parallel:result is None 2024-07-28T19:50:26.343 INFO:teuthology.orchestra.run.smithi100.stdout:167503724867 2024-07-28T19:50:26.610 INFO:teuthology.orchestra.run.smithi100.stdout:2761663971427 2024-07-28T19:50:26.951 INFO:teuthology.orchestra.run.smithi100.stdout:2697239461993 2024-07-28T19:50:27.187 INFO:teuthology.orchestra.run.smithi100.stdout:2800318677085 2024-07-28T19:50:27.534 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[82644]: cluster 2024-07-28T19:50:26.103423+0000 mgr.y (mgr.25358) 535 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:27.535 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[82644]: audit 2024-07-28T19:50:26.345958+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.100:0/1426461900' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:50:27.535 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[82644]: audit 2024-07-28T19:50:26.612311+0000 mon.a (mon.0) 1552 : audit [DBG] from='client.? 172.21.15.100:0/289436931' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:50:27.535 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[82644]: audit 2024-07-28T19:50:26.953508+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.100:0/2694405318' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:50:27.535 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[82644]: audit 2024-07-28T19:50:27.188979+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.100:0/430734120' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:50:27.535 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[80199]: cluster 2024-07-28T19:50:26.103423+0000 mgr.y (mgr.25358) 535 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:27.535 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[80199]: audit 2024-07-28T19:50:26.345958+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.100:0/1426461900' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:50:27.535 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[80199]: audit 2024-07-28T19:50:26.612311+0000 mon.a (mon.0) 1552 : audit [DBG] from='client.? 172.21.15.100:0/289436931' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:50:27.535 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[80199]: audit 2024-07-28T19:50:26.953508+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.100:0/2694405318' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:50:27.536 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:27 smithi100 bash[80199]: audit 2024-07-28T19:50:27.188979+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.100:0/430734120' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:50:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:27 smithi138 bash[67894]: cluster 2024-07-28T19:50:26.103423+0000 mgr.y (mgr.25358) 535 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:27 smithi138 bash[67894]: audit 2024-07-28T19:50:26.345958+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.100:0/1426461900' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-07-28T19:50:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:27 smithi138 bash[67894]: audit 2024-07-28T19:50:26.612311+0000 mon.a (mon.0) 1552 : audit [DBG] from='client.? 172.21.15.100:0/289436931' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-07-28T19:50:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:27 smithi138 bash[67894]: audit 2024-07-28T19:50:26.953508+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.100:0/2694405318' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-07-28T19:50:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:27 smithi138 bash[67894]: audit 2024-07-28T19:50:27.188979+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.100:0/430734120' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-07-28T19:50:28.411 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724863 got 167503724867 for osd.5 2024-07-28T19:50:28.411 DEBUG:teuthology.parallel:result is None 2024-07-28T19:50:28.575 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2761663971422 got 2761663971427 for osd.3 2024-07-28T19:50:28.575 DEBUG:teuthology.parallel:result is None 2024-07-28T19:50:28.824 INFO:teuthology.orchestra.run.smithi100.stdout:2319282339964 2024-07-28T19:50:29.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:29 smithi100 bash[80199]: cluster 2024-07-28T19:50:28.104155+0000 mgr.y (mgr.25358) 536 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:29.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:29 smithi100 bash[80199]: audit 2024-07-28T19:50:28.826005+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.100:0/1814952863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:50:29.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:29 smithi100 bash[82644]: cluster 2024-07-28T19:50:28.104155+0000 mgr.y (mgr.25358) 536 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:29.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:29 smithi100 bash[82644]: audit 2024-07-28T19:50:28.826005+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.100:0/1814952863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:50:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:29 smithi138 bash[67894]: cluster 2024-07-28T19:50:28.104155+0000 mgr.y (mgr.25358) 536 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:29.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:29 smithi138 bash[67894]: audit 2024-07-28T19:50:28.826005+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.100:0/1814952863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-07-28T19:50:30.259 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2697239461989 got 2697239461993 for osd.2 2024-07-28T19:50:30.259 DEBUG:teuthology.parallel:result is None 2024-07-28T19:50:30.381 INFO:teuthology.orchestra.run.smithi100.stdout:201863463226 2024-07-28T19:50:30.472 INFO:teuthology.orchestra.run.smithi100.stdout:2173253451907 2024-07-28T19:50:30.617 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2800318677081 got 2800318677085 for osd.4 2024-07-28T19:50:30.617 DEBUG:teuthology.parallel:result is None 2024-07-28T19:50:31.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:31 smithi100 bash[82644]: cluster 2024-07-28T19:50:30.104822+0000 mgr.y (mgr.25358) 537 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:31.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:31 smithi100 bash[82644]: audit 2024-07-28T19:50:30.383443+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.100:0/3338609671' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:50:31.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:31 smithi100 bash[82644]: audit 2024-07-28T19:50:30.474591+0000 mon.a (mon.0) 1555 : audit [DBG] from='client.? 172.21.15.100:0/392489558' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:50:31.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:31 smithi100 bash[80199]: cluster 2024-07-28T19:50:30.104822+0000 mgr.y (mgr.25358) 537 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:31.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:31 smithi100 bash[80199]: audit 2024-07-28T19:50:30.383443+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.100:0/3338609671' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:50:31.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:31 smithi100 bash[80199]: audit 2024-07-28T19:50:30.474591+0000 mon.a (mon.0) 1555 : audit [DBG] from='client.? 172.21.15.100:0/392489558' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:50:31.505 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2319282339959 got 2319282339964 for osd.1 2024-07-28T19:50:31.505 DEBUG:teuthology.parallel:result is None 2024-07-28T19:50:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:31 smithi138 bash[67894]: cluster 2024-07-28T19:50:30.104822+0000 mgr.y (mgr.25358) 537 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:31 smithi138 bash[67894]: audit 2024-07-28T19:50:30.383443+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.100:0/3338609671' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-07-28T19:50:31.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:31 smithi138 bash[67894]: audit 2024-07-28T19:50:30.474591+0000 mon.a (mon.0) 1555 : audit [DBG] from='client.? 172.21.15.100:0/392489558' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-07-28T19:50:32.286 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463221 got 201863463226 for osd.6 2024-07-28T19:50:32.286 DEBUG:teuthology.parallel:result is None 2024-07-28T19:50:32.301 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2173253451903 got 2173253451907 for osd.0 2024-07-28T19:50:32.302 DEBUG:teuthology.parallel:result is None 2024-07-28T19:50:32.302 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-07-28T19:50:32.302 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph pg dump --format=json 2024-07-28T19:50:32.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:50:32 smithi138 bash[60615]: ts=2024-07-28T19:50:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:50:33.232 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:50:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:50:32] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-07-28T19:50:33.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:33 smithi100 bash[80199]: cluster 2024-07-28T19:50:32.105482+0000 mgr.y (mgr.25358) 538 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:33.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:33 smithi100 bash[82644]: cluster 2024-07-28T19:50:32.105482+0000 mgr.y (mgr.25358) 538 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:33.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:33 smithi138 bash[67894]: cluster 2024-07-28T19:50:32.105482+0000 mgr.y (mgr.25358) 538 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:35.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:35 smithi100 bash[80199]: cluster 2024-07-28T19:50:34.106397+0000 mgr.y (mgr.25358) 539 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:35.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:35 smithi100 bash[82644]: cluster 2024-07-28T19:50:34.106397+0000 mgr.y (mgr.25358) 539 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:35.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:35 smithi138 bash[67894]: cluster 2024-07-28T19:50:34.106397+0000 mgr.y (mgr.25358) 539 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:37.127 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:37 smithi100 bash[80199]: cluster 2024-07-28T19:50:36.107260+0000 mgr.y (mgr.25358) 540 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:37.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:37 smithi100 bash[82644]: cluster 2024-07-28T19:50:36.107260+0000 mgr.y (mgr.25358) 540 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:37.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:37 smithi138 bash[67894]: cluster 2024-07-28T19:50:36.107260+0000 mgr.y (mgr.25358) 540 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:38.511 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:38 smithi100 bash[80199]: audit 2024-07-28T19:50:37.647430+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:38.511 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:38 smithi100 bash[82644]: audit 2024-07-28T19:50:37.647430+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:38.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:38 smithi138 bash[67894]: audit 2024-07-28T19:50:37.647430+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:39.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:39 smithi138 bash[67894]: cluster 2024-07-28T19:50:38.107992+0000 mgr.y (mgr.25358) 541 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:39.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:39 smithi100 bash[80199]: cluster 2024-07-28T19:50:38.107992+0000 mgr.y (mgr.25358) 541 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:39.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:39 smithi100 bash[82644]: cluster 2024-07-28T19:50:38.107992+0000 mgr.y (mgr.25358) 541 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:40.781 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:50:40.782 INFO:teuthology.orchestra.run.smithi100.stderr:dumped all 2024-07-28T19:50:41.508 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:41 smithi100 bash[82644]: cluster 2024-07-28T19:50:40.108971+0000 mgr.y (mgr.25358) 542 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:41.508 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:41 smithi100 bash[80199]: cluster 2024-07-28T19:50:40.108971+0000 mgr.y (mgr.25358) 542 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:41.509 INFO:teuthology.orchestra.run.smithi100.stdout:{"pg_ready":true,"pg_map":{"version":1112,"stamp":"2024-07-28T19:50:40.108293+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595910,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":538,"num_read_kb":964,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1315644,"kb_used_data":89580,"kb_used_omap":318,"kb_used_meta":1173761,"kb_avail":748645572,"statfs":{"total":767960285184,"available":766613065728,"internally_reserved":0,"allocated":91729920,"data_stored":59110330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":325982,"internal_metadata":1201931938},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004594"},"pg_stats":[{"pgid":"3.1f","version":"70'5","reported_seq":912,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.030379+0000","last_change":"2024-07-28T19:41:50.992404+0000","last_active":"2024-07-28T19:49:16.030379+0000","last_peered":"2024-07-28T19:49:16.030379+0000","last_clean":"2024-07-28T19:49:16.030379+0000","last_became_active":"2024-07-28T19:41:50.991667+0000","last_became_peered":"2024-07-28T19:41:50.991667+0000","last_unstale":"2024-07-28T19:49:16.030379+0000","last_undegraded":"2024-07-28T19:49:16.030379+0000","last_fullsized":"2024-07-28T19:49:16.030379+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:43:38.150311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":926,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.298309+0000","last_change":"2024-07-28T19:42:51.406228+0000","last_active":"2024-07-28T19:49:15.298309+0000","last_peered":"2024-07-28T19:49:15.298309+0000","last_clean":"2024-07-28T19:49:15.298309+0000","last_became_active":"2024-07-28T19:42:51.398223+0000","last_became_peered":"2024-07-28T19:42:51.398223+0000","last_unstale":"2024-07-28T19:49:15.298309+0000","last_undegraded":"2024-07-28T19:49:15.298309+0000","last_fullsized":"2024-07-28T19:49:15.298309+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:21:35.052391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":917,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.293766+0000","last_change":"2024-07-28T19:42:19.269168+0000","last_active":"2024-07-28T19:49:15.293766+0000","last_peered":"2024-07-28T19:49:15.293766+0000","last_clean":"2024-07-28T19:49:15.293766+0000","last_became_active":"2024-07-28T19:42:19.268312+0000","last_became_peered":"2024-07-28T19:42:19.268312+0000","last_unstale":"2024-07-28T19:49:15.293766+0000","last_undegraded":"2024-07-28T19:49:15.293766+0000","last_fullsized":"2024-07-28T19:49:15.293766+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:50:25.364241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":930,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.291650+0000","last_change":"2024-07-28T19:42:19.243100+0000","last_active":"2024-07-28T19:49:15.291650+0000","last_peered":"2024-07-28T19:49:15.291650+0000","last_clean":"2024-07-28T19:49:15.291650+0000","last_became_active":"2024-07-28T19:42:19.242831+0000","last_became_peered":"2024-07-28T19:42:19.242831+0000","last_unstale":"2024-07-28T19:49:15.291650+0000","last_undegraded":"2024-07-28T19:49:15.291650+0000","last_fullsized":"2024-07-28T19:49:15.291650+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:09:12.249965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":914,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.416932+0000","last_change":"2024-07-28T19:42:51.419502+0000","last_active":"2024-07-28T19:49:14.416932+0000","last_peered":"2024-07-28T19:49:14.416932+0000","last_clean":"2024-07-28T19:49:14.416932+0000","last_became_active":"2024-07-28T19:42:51.419110+0000","last_became_peered":"2024-07-28T19:42:51.419110+0000","last_unstale":"2024-07-28T19:49:14.416932+0000","last_undegraded":"2024-07-28T19:49:14.416932+0000","last_fullsized":"2024-07-28T19:49:14.416932+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:24:21.005730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":940,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.300018+0000","last_change":"2024-07-28T19:42:51.401545+0000","last_active":"2024-07-28T19:49:15.300018+0000","last_peered":"2024-07-28T19:49:15.300018+0000","last_clean":"2024-07-28T19:49:15.300018+0000","last_became_active":"2024-07-28T19:42:51.400879+0000","last_became_peered":"2024-07-28T19:42:51.400879+0000","last_unstale":"2024-07-28T19:49:15.300018+0000","last_undegraded":"2024-07-28T19:49:15.300018+0000","last_fullsized":"2024-07-28T19:49:15.300018+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:22:22.537822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":931,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.416084+0000","last_change":"2024-07-28T19:42:51.421716+0000","last_active":"2024-07-28T19:49:14.416084+0000","last_peered":"2024-07-28T19:49:14.416084+0000","last_clean":"2024-07-28T19:49:14.416084+0000","last_became_active":"2024-07-28T19:42:51.421372+0000","last_became_peered":"2024-07-28T19:42:51.421372+0000","last_unstale":"2024-07-28T19:49:14.416084+0000","last_undegraded":"2024-07-28T19:49:14.416084+0000","last_fullsized":"2024-07-28T19:49:14.416084+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:29:04.117086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":940,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.303431+0000","last_change":"2024-07-28T19:42:51.425099+0000","last_active":"2024-07-28T19:49:15.303431+0000","last_peered":"2024-07-28T19:49:15.303431+0000","last_clean":"2024-07-28T19:49:15.303431+0000","last_became_active":"2024-07-28T19:42:51.420828+0000","last_became_peered":"2024-07-28T19:42:51.420828+0000","last_unstale":"2024-07-28T19:49:15.303431+0000","last_undegraded":"2024-07-28T19:49:15.303431+0000","last_fullsized":"2024-07-28T19:49:15.303431+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:43:09.431596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":906,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.027218+0000","last_change":"2024-07-28T19:42:51.433065+0000","last_active":"2024-07-28T19:49:16.027218+0000","last_peered":"2024-07-28T19:49:16.027218+0000","last_clean":"2024-07-28T19:49:16.027218+0000","last_became_active":"2024-07-28T19:42:51.432308+0000","last_became_peered":"2024-07-28T19:42:51.432308+0000","last_unstale":"2024-07-28T19:49:16.027218+0000","last_undegraded":"2024-07-28T19:49:16.027218+0000","last_fullsized":"2024-07-28T19:49:16.027218+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:54:54.488258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":896,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.689335+0000","last_change":"2024-07-28T19:42:19.241821+0000","last_active":"2024-07-28T19:49:15.689335+0000","last_peered":"2024-07-28T19:49:15.689335+0000","last_clean":"2024-07-28T19:49:15.689335+0000","last_became_active":"2024-07-28T19:42:19.241459+0000","last_became_peered":"2024-07-28T19:42:19.241459+0000","last_unstale":"2024-07-28T19:49:15.689335+0000","last_undegraded":"2024-07-28T19:49:15.689335+0000","last_fullsized":"2024-07-28T19:49:15.689335+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:58:56.626908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":946,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.300501+0000","last_change":"2024-07-28T19:42:51.408616+0000","last_active":"2024-07-28T19:49:15.300501+0000","last_peered":"2024-07-28T19:49:15.300501+0000","last_clean":"2024-07-28T19:49:15.300501+0000","last_became_active":"2024-07-28T19:42:51.405947+0000","last_became_peered":"2024-07-28T19:42:51.405947+0000","last_unstale":"2024-07-28T19:49:15.300501+0000","last_undegraded":"2024-07-28T19:49:15.300501+0000","last_fullsized":"2024-07-28T19:49:15.300501+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:53:22.106233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":923,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.294393+0000","last_change":"2024-07-28T19:42:51.441869+0000","last_active":"2024-07-28T19:49:15.294393+0000","last_peered":"2024-07-28T19:49:15.294393+0000","last_clean":"2024-07-28T19:49:15.294393+0000","last_became_active":"2024-07-28T19:42:51.441528+0000","last_became_peered":"2024-07-28T19:42:51.441528+0000","last_unstale":"2024-07-28T19:49:15.294393+0000","last_undegraded":"2024-07-28T19:49:15.294393+0000","last_fullsized":"2024-07-28T19:49:15.294393+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:27:32.962528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":921,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.733890+0000","last_change":"2024-07-28T19:42:51.381104+0000","last_active":"2024-07-28T19:49:15.733890+0000","last_peered":"2024-07-28T19:49:15.733890+0000","last_clean":"2024-07-28T19:49:15.733890+0000","last_became_active":"2024-07-28T19:42:51.380818+0000","last_became_peered":"2024-07-28T19:42:51.380818+0000","last_unstale":"2024-07-28T19:49:15.733890+0000","last_undegraded":"2024-07-28T19:49:15.733890+0000","last_fullsized":"2024-07-28T19:49:15.733890+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:09:25.890587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":902,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.028523+0000","last_change":"2024-07-28T19:42:19.274570+0000","last_active":"2024-07-28T19:49:16.028523+0000","last_peered":"2024-07-28T19:49:16.028523+0000","last_clean":"2024-07-28T19:49:16.028523+0000","last_became_active":"2024-07-28T19:42:19.273406+0000","last_became_peered":"2024-07-28T19:42:19.273406+0000","last_unstale":"2024-07-28T19:49:16.028523+0000","last_undegraded":"2024-07-28T19:49:16.028523+0000","last_fullsized":"2024-07-28T19:49:16.028523+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:07:20.283774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":926,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.291348+0000","last_change":"2024-07-28T19:42:19.241728+0000","last_active":"2024-07-28T19:49:15.291348+0000","last_peered":"2024-07-28T19:49:15.291348+0000","last_clean":"2024-07-28T19:49:15.291348+0000","last_became_active":"2024-07-28T19:42:19.241448+0000","last_became_peered":"2024-07-28T19:42:19.241448+0000","last_unstale":"2024-07-28T19:49:15.291348+0000","last_undegraded":"2024-07-28T19:49:15.291348+0000","last_fullsized":"2024-07-28T19:49:15.291348+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:01:23.574357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":910,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.419553+0000","last_change":"2024-07-28T19:40:16.070038+0000","last_active":"2024-07-28T19:49:14.419553+0000","last_peered":"2024-07-28T19:49:14.419553+0000","last_clean":"2024-07-28T19:49:14.419553+0000","last_became_active":"2024-07-28T19:40:16.069305+0000","last_became_peered":"2024-07-28T19:40:16.069305+0000","last_unstale":"2024-07-28T19:49:14.419553+0000","last_undegraded":"2024-07-28T19:49:14.419553+0000","last_fullsized":"2024-07-28T19:49:14.419553+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T06:20:30.019331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":908,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.291797+0000","last_change":"2024-07-28T19:42:19.266717+0000","last_active":"2024-07-28T19:49:15.291797+0000","last_peered":"2024-07-28T19:49:15.291797+0000","last_clean":"2024-07-28T19:49:15.291797+0000","last_became_active":"2024-07-28T19:42:19.266351+0000","last_became_peered":"2024-07-28T19:42:19.266351+0000","last_unstale":"2024-07-28T19:49:15.291797+0000","last_undegraded":"2024-07-28T19:49:15.291797+0000","last_fullsized":"2024-07-28T19:49:15.291797+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T06:56:16.625754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":900,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.027248+0000","last_change":"2024-07-28T19:42:19.263249+0000","last_active":"2024-07-28T19:49:16.027248+0000","last_peered":"2024-07-28T19:49:16.027248+0000","last_clean":"2024-07-28T19:49:16.027248+0000","last_became_active":"2024-07-28T19:42:19.262846+0000","last_became_peered":"2024-07-28T19:42:19.262846+0000","last_unstale":"2024-07-28T19:49:16.027248+0000","last_undegraded":"2024-07-28T19:49:16.027248+0000","last_fullsized":"2024-07-28T19:49:16.027248+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:56:59.694417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":869,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.291845+0000","last_change":"2024-07-28T19:42:19.230447+0000","last_active":"2024-07-28T19:49:15.291845+0000","last_peered":"2024-07-28T19:49:15.291845+0000","last_clean":"2024-07-28T19:49:15.291845+0000","last_became_active":"2024-07-28T19:42:19.229707+0000","last_became_peered":"2024-07-28T19:42:19.229707+0000","last_unstale":"2024-07-28T19:49:15.291845+0000","last_undegraded":"2024-07-28T19:49:15.291845+0000","last_fullsized":"2024-07-28T19:49:15.291845+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:03:22.965514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":911,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.416623+0000","last_change":"2024-07-28T19:40:16.062070+0000","last_active":"2024-07-28T19:49:14.416623+0000","last_peered":"2024-07-28T19:49:14.416623+0000","last_clean":"2024-07-28T19:49:14.416623+0000","last_became_active":"2024-07-28T19:40:16.061253+0000","last_became_peered":"2024-07-28T19:40:16.061253+0000","last_unstale":"2024-07-28T19:49:14.416623+0000","last_undegraded":"2024-07-28T19:49:14.416623+0000","last_fullsized":"2024-07-28T19:49:14.416623+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:09:42.378599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":935,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411877+0000","last_change":"2024-07-28T19:42:51.425444+0000","last_active":"2024-07-28T19:49:14.411877+0000","last_peered":"2024-07-28T19:49:14.411877+0000","last_clean":"2024-07-28T19:49:14.411877+0000","last_became_active":"2024-07-28T19:42:51.424962+0000","last_became_peered":"2024-07-28T19:42:51.424962+0000","last_unstale":"2024-07-28T19:49:14.411877+0000","last_undegraded":"2024-07-28T19:49:14.411877+0000","last_fullsized":"2024-07-28T19:49:14.411877+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:12:35.969954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":906,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.736508+0000","last_change":"2024-07-28T19:41:51.007682+0000","last_active":"2024-07-28T19:49:15.736508+0000","last_peered":"2024-07-28T19:49:15.736508+0000","last_clean":"2024-07-28T19:49:15.736508+0000","last_became_active":"2024-07-28T19:41:51.007086+0000","last_became_peered":"2024-07-28T19:41:51.007086+0000","last_unstale":"2024-07-28T19:49:15.736508+0000","last_undegraded":"2024-07-28T19:49:15.736508+0000","last_fullsized":"2024-07-28T19:49:15.736508+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:14:39.168420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"379'23","reported_seq":917,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.300264+0000","last_change":"2024-07-28T19:42:51.421626+0000","last_active":"2024-07-28T19:49:15.300264+0000","last_peered":"2024-07-28T19:49:15.300264+0000","last_clean":"2024-07-28T19:49:15.300264+0000","last_became_active":"2024-07-28T19:42:51.418634+0000","last_became_peered":"2024-07-28T19:42:51.418634+0000","last_unstale":"2024-07-28T19:49:15.300264+0000","last_undegraded":"2024-07-28T19:49:15.300264+0000","last_fullsized":"2024-07-28T19:49:15.300264+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:06:51.752498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":903,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.303038+0000","last_change":"2024-07-28T19:42:51.425291+0000","last_active":"2024-07-28T19:49:15.303038+0000","last_peered":"2024-07-28T19:49:15.303038+0000","last_clean":"2024-07-28T19:49:15.303038+0000","last_became_active":"2024-07-28T19:42:51.421933+0000","last_became_peered":"2024-07-28T19:42:51.421933+0000","last_unstale":"2024-07-28T19:49:15.303038+0000","last_undegraded":"2024-07-28T19:49:15.303038+0000","last_fullsized":"2024-07-28T19:49:15.303038+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:32:25.468432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":909,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.413024+0000","last_change":"2024-07-28T19:42:51.398708+0000","last_active":"2024-07-28T19:49:14.413024+0000","last_peered":"2024-07-28T19:49:14.413024+0000","last_clean":"2024-07-28T19:49:14.413024+0000","last_became_active":"2024-07-28T19:42:51.394596+0000","last_became_peered":"2024-07-28T19:42:51.394596+0000","last_unstale":"2024-07-28T19:49:14.413024+0000","last_undegraded":"2024-07-28T19:49:14.413024+0000","last_fullsized":"2024-07-28T19:49:14.413024+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:30:35.750535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":915,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.419706+0000","last_change":"2024-07-28T19:41:50.981938+0000","last_active":"2024-07-28T19:49:14.419706+0000","last_peered":"2024-07-28T19:49:14.419706+0000","last_clean":"2024-07-28T19:49:14.419706+0000","last_became_active":"2024-07-28T19:41:50.981204+0000","last_became_peered":"2024-07-28T19:41:50.981204+0000","last_unstale":"2024-07-28T19:49:14.419706+0000","last_undegraded":"2024-07-28T19:49:14.419706+0000","last_fullsized":"2024-07-28T19:49:14.419706+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:32:36.925368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":921,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.029204+0000","last_change":"2024-07-28T19:42:51.379059+0000","last_active":"2024-07-28T19:49:16.029204+0000","last_peered":"2024-07-28T19:49:16.029204+0000","last_clean":"2024-07-28T19:49:16.029204+0000","last_became_active":"2024-07-28T19:42:51.378354+0000","last_became_peered":"2024-07-28T19:42:51.378354+0000","last_unstale":"2024-07-28T19:49:16.029204+0000","last_undegraded":"2024-07-28T19:49:16.029204+0000","last_fullsized":"2024-07-28T19:49:16.029204+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:51:45.854172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":915,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.292004+0000","last_change":"2024-07-28T19:42:19.278084+0000","last_active":"2024-07-28T19:49:15.292004+0000","last_peered":"2024-07-28T19:49:15.292004+0000","last_clean":"2024-07-28T19:49:15.292004+0000","last_became_active":"2024-07-28T19:42:19.277738+0000","last_became_peered":"2024-07-28T19:42:19.277738+0000","last_unstale":"2024-07-28T19:49:15.292004+0000","last_undegraded":"2024-07-28T19:49:15.292004+0000","last_fullsized":"2024-07-28T19:49:15.292004+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:21:44.055358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":907,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.424176+0000","last_change":"2024-07-28T19:42:51.382107+0000","last_active":"2024-07-28T19:49:14.424176+0000","last_peered":"2024-07-28T19:49:14.424176+0000","last_clean":"2024-07-28T19:49:14.424176+0000","last_became_active":"2024-07-28T19:42:51.381833+0000","last_became_peered":"2024-07-28T19:42:51.381833+0000","last_unstale":"2024-07-28T19:49:14.424176+0000","last_undegraded":"2024-07-28T19:49:14.424176+0000","last_fullsized":"2024-07-28T19:49:14.424176+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:01:56.634928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":918,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.429434+0000","last_change":"2024-07-28T19:42:19.284578+0000","last_active":"2024-07-28T19:49:14.429434+0000","last_peered":"2024-07-28T19:49:14.429434+0000","last_clean":"2024-07-28T19:49:14.429434+0000","last_became_active":"2024-07-28T19:42:19.279079+0000","last_became_peered":"2024-07-28T19:42:19.279079+0000","last_unstale":"2024-07-28T19:49:14.429434+0000","last_undegraded":"2024-07-28T19:49:14.429434+0000","last_fullsized":"2024-07-28T19:49:14.429434+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:39:16.242525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":887,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.028525+0000","last_change":"2024-07-28T19:39:42.015588+0000","last_active":"2024-07-28T19:49:16.028525+0000","last_peered":"2024-07-28T19:49:16.028525+0000","last_clean":"2024-07-28T19:49:16.028525+0000","last_became_active":"2024-07-28T19:39:42.013634+0000","last_became_peered":"2024-07-28T19:39:42.013634+0000","last_unstale":"2024-07-28T19:49:16.028525+0000","last_undegraded":"2024-07-28T19:49:16.028525+0000","last_fullsized":"2024-07-28T19:49:16.028525+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:35:28.512750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":934,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.413610+0000","last_change":"2024-07-28T19:42:51.411817+0000","last_active":"2024-07-28T19:49:14.413610+0000","last_peered":"2024-07-28T19:49:14.413610+0000","last_clean":"2024-07-28T19:49:14.413610+0000","last_became_active":"2024-07-28T19:42:51.411521+0000","last_became_peered":"2024-07-28T19:42:51.411521+0000","last_unstale":"2024-07-28T19:49:14.413610+0000","last_undegraded":"2024-07-28T19:49:14.413610+0000","last_fullsized":"2024-07-28T19:49:14.413610+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:55:04.229692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":907,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.737804+0000","last_change":"2024-07-28T19:41:51.004987+0000","last_active":"2024-07-28T19:49:15.737804+0000","last_peered":"2024-07-28T19:49:15.737804+0000","last_clean":"2024-07-28T19:49:15.737804+0000","last_became_active":"2024-07-28T19:41:51.004670+0000","last_became_peered":"2024-07-28T19:41:51.004670+0000","last_unstale":"2024-07-28T19:49:15.737804+0000","last_undegraded":"2024-07-28T19:49:15.737804+0000","last_fullsized":"2024-07-28T19:49:15.737804+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:44:12.419613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":899,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.688723+0000","last_change":"2024-07-28T19:42:51.421653+0000","last_active":"2024-07-28T19:49:15.688723+0000","last_peered":"2024-07-28T19:49:15.688723+0000","last_clean":"2024-07-28T19:49:15.688723+0000","last_became_active":"2024-07-28T19:42:51.421366+0000","last_became_peered":"2024-07-28T19:42:51.421366+0000","last_unstale":"2024-07-28T19:49:15.688723+0000","last_undegraded":"2024-07-28T19:49:15.688723+0000","last_fullsized":"2024-07-28T19:49:15.688723+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:59:56.862003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":902,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.737710+0000","last_change":"2024-07-28T19:40:16.080446+0000","last_active":"2024-07-28T19:49:15.737710+0000","last_peered":"2024-07-28T19:49:15.737710+0000","last_clean":"2024-07-28T19:49:15.737710+0000","last_became_active":"2024-07-28T19:40:16.079427+0000","last_became_peered":"2024-07-28T19:40:16.079427+0000","last_unstale":"2024-07-28T19:49:15.737710+0000","last_undegraded":"2024-07-28T19:49:15.737710+0000","last_fullsized":"2024-07-28T19:49:15.737710+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:01:44.248862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":928,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.416759+0000","last_change":"2024-07-28T19:42:19.270718+0000","last_active":"2024-07-28T19:49:14.416759+0000","last_peered":"2024-07-28T19:49:14.416759+0000","last_clean":"2024-07-28T19:49:14.416759+0000","last_became_active":"2024-07-28T19:42:19.270277+0000","last_became_peered":"2024-07-28T19:42:19.270277+0000","last_unstale":"2024-07-28T19:49:14.416759+0000","last_undegraded":"2024-07-28T19:49:14.416759+0000","last_fullsized":"2024-07-28T19:49:14.416759+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:52:06.646794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":922,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.737917+0000","last_change":"2024-07-28T19:42:19.258262+0000","last_active":"2024-07-28T19:49:15.737917+0000","last_peered":"2024-07-28T19:49:15.737917+0000","last_clean":"2024-07-28T19:49:15.737917+0000","last_became_active":"2024-07-28T19:42:19.257951+0000","last_became_peered":"2024-07-28T19:42:19.257951+0000","last_unstale":"2024-07-28T19:49:15.737917+0000","last_undegraded":"2024-07-28T19:49:15.737917+0000","last_fullsized":"2024-07-28T19:49:15.737917+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:58:19.686714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":911,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.415731+0000","last_change":"2024-07-28T19:42:51.425299+0000","last_active":"2024-07-28T19:49:14.415731+0000","last_peered":"2024-07-28T19:49:14.415731+0000","last_clean":"2024-07-28T19:49:14.415731+0000","last_became_active":"2024-07-28T19:42:51.421774+0000","last_became_peered":"2024-07-28T19:42:51.421774+0000","last_unstale":"2024-07-28T19:49:14.415731+0000","last_undegraded":"2024-07-28T19:49:14.415731+0000","last_fullsized":"2024-07-28T19:49:14.415731+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:18:26.859879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":915,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411546+0000","last_change":"2024-07-28T19:40:16.078698+0000","last_active":"2024-07-28T19:49:14.411546+0000","last_peered":"2024-07-28T19:49:14.411546+0000","last_clean":"2024-07-28T19:49:14.411546+0000","last_became_active":"2024-07-28T19:40:16.077852+0000","last_became_peered":"2024-07-28T19:40:16.077852+0000","last_unstale":"2024-07-28T19:49:14.411546+0000","last_undegraded":"2024-07-28T19:49:14.411546+0000","last_fullsized":"2024-07-28T19:49:14.411546+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:26:06.111600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":896,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.424508+0000","last_change":"2024-07-28T19:40:16.066113+0000","last_active":"2024-07-28T19:49:14.424508+0000","last_peered":"2024-07-28T19:49:14.424508+0000","last_clean":"2024-07-28T19:49:14.424508+0000","last_became_active":"2024-07-28T19:40:16.065767+0000","last_became_peered":"2024-07-28T19:40:16.065767+0000","last_unstale":"2024-07-28T19:49:14.424508+0000","last_undegraded":"2024-07-28T19:49:14.424508+0000","last_fullsized":"2024-07-28T19:49:14.424508+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:18:45.882295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":907,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.734117+0000","last_change":"2024-07-28T19:41:51.001335+0000","last_active":"2024-07-28T19:49:15.734117+0000","last_peered":"2024-07-28T19:49:15.734117+0000","last_clean":"2024-07-28T19:49:15.734117+0000","last_became_active":"2024-07-28T19:41:51.000000+0000","last_became_peered":"2024-07-28T19:41:51.000000+0000","last_unstale":"2024-07-28T19:49:15.734117+0000","last_undegraded":"2024-07-28T19:49:15.734117+0000","last_fullsized":"2024-07-28T19:49:15.734117+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:08:38.692771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":909,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.688996+0000","last_change":"2024-07-28T19:42:19.238407+0000","last_active":"2024-07-28T19:49:15.688996+0000","last_peered":"2024-07-28T19:49:15.688996+0000","last_clean":"2024-07-28T19:49:15.688996+0000","last_became_active":"2024-07-28T19:42:19.238024+0000","last_became_peered":"2024-07-28T19:42:19.238024+0000","last_unstale":"2024-07-28T19:49:15.688996+0000","last_undegraded":"2024-07-28T19:49:15.688996+0000","last_fullsized":"2024-07-28T19:49:15.688996+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:27:02.041697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":916,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.030435+0000","last_change":"2024-07-28T19:41:50.993084+0000","last_active":"2024-07-28T19:49:16.030435+0000","last_peered":"2024-07-28T19:49:16.030435+0000","last_clean":"2024-07-28T19:49:16.030435+0000","last_became_active":"2024-07-28T19:41:50.992504+0000","last_became_peered":"2024-07-28T19:41:50.992504+0000","last_unstale":"2024-07-28T19:49:16.030435+0000","last_undegraded":"2024-07-28T19:49:16.030435+0000","last_fullsized":"2024-07-28T19:49:16.030435+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:17:34.418222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":916,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.027040+0000","last_change":"2024-07-28T19:42:51.392011+0000","last_active":"2024-07-28T19:49:16.027040+0000","last_peered":"2024-07-28T19:49:16.027040+0000","last_clean":"2024-07-28T19:49:16.027040+0000","last_became_active":"2024-07-28T19:42:51.391619+0000","last_became_peered":"2024-07-28T19:42:51.391619+0000","last_unstale":"2024-07-28T19:49:16.027040+0000","last_undegraded":"2024-07-28T19:49:16.027040+0000","last_fullsized":"2024-07-28T19:49:16.027040+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:52:41.939340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":934,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.414256+0000","last_change":"2024-07-28T19:40:16.069978+0000","last_active":"2024-07-28T19:49:14.414256+0000","last_peered":"2024-07-28T19:49:14.414256+0000","last_clean":"2024-07-28T19:49:14.414256+0000","last_became_active":"2024-07-28T19:40:16.068975+0000","last_became_peered":"2024-07-28T19:40:16.068975+0000","last_unstale":"2024-07-28T19:49:14.414256+0000","last_undegraded":"2024-07-28T19:49:14.414256+0000","last_fullsized":"2024-07-28T19:49:14.414256+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:17:58.077529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":910,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.419568+0000","last_change":"2024-07-28T19:42:51.416800+0000","last_active":"2024-07-28T19:49:14.419568+0000","last_peered":"2024-07-28T19:49:14.419568+0000","last_clean":"2024-07-28T19:49:14.419568+0000","last_became_active":"2024-07-28T19:42:51.416450+0000","last_became_peered":"2024-07-28T19:42:51.416450+0000","last_unstale":"2024-07-28T19:49:14.419568+0000","last_undegraded":"2024-07-28T19:49:14.419568+0000","last_fullsized":"2024-07-28T19:49:14.419568+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:58:51.835822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":869,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.299724+0000","last_change":"2024-07-28T19:42:19.269576+0000","last_active":"2024-07-28T19:49:15.299724+0000","last_peered":"2024-07-28T19:49:15.299724+0000","last_clean":"2024-07-28T19:49:15.299724+0000","last_became_active":"2024-07-28T19:42:19.266463+0000","last_became_peered":"2024-07-28T19:42:19.266463+0000","last_unstale":"2024-07-28T19:49:15.299724+0000","last_undegraded":"2024-07-28T19:49:15.299724+0000","last_fullsized":"2024-07-28T19:49:15.299724+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:00:16.617549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":915,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411986+0000","last_change":"2024-07-28T19:41:51.000919+0000","last_active":"2024-07-28T19:49:14.411986+0000","last_peered":"2024-07-28T19:49:14.411986+0000","last_clean":"2024-07-28T19:49:14.411986+0000","last_became_active":"2024-07-28T19:41:51.000578+0000","last_became_peered":"2024-07-28T19:41:51.000578+0000","last_unstale":"2024-07-28T19:49:14.411986+0000","last_undegraded":"2024-07-28T19:49:14.411986+0000","last_fullsized":"2024-07-28T19:49:14.411986+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:09:59.009367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"305'161","reported_seq":1293,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.413810+0000","last_change":"2024-07-28T19:39:42.005813+0000","last_active":"2024-07-28T19:49:14.413810+0000","last_peered":"2024-07-28T19:49:14.413810+0000","last_clean":"2024-07-28T19:49:14.413810+0000","last_became_active":"2024-07-28T19:39:42.004739+0000","last_became_peered":"2024-07-28T19:39:42.004739+0000","last_unstale":"2024-07-28T19:49:14.413810+0000","last_undegraded":"2024-07-28T19:49:14.413810+0000","last_fullsized":"2024-07-28T19:49:14.413810+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_clean_scrub_stamp":"2024-07-28T19:21:20.215133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:25:39.147946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":921,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.424813+0000","last_change":"2024-07-28T19:42:51.399069+0000","last_active":"2024-07-28T19:49:14.424813+0000","last_peered":"2024-07-28T19:49:14.424813+0000","last_clean":"2024-07-28T19:49:14.424813+0000","last_became_active":"2024-07-28T19:42:51.398615+0000","last_became_peered":"2024-07-28T19:42:51.398615+0000","last_unstale":"2024-07-28T19:49:14.424813+0000","last_undegraded":"2024-07-28T19:49:14.424813+0000","last_fullsized":"2024-07-28T19:49:14.424813+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:20:40.672584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":909,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.427640+0000","last_change":"2024-07-28T19:42:19.278426+0000","last_active":"2024-07-28T19:49:14.427640+0000","last_peered":"2024-07-28T19:49:14.427640+0000","last_clean":"2024-07-28T19:49:14.427640+0000","last_became_active":"2024-07-28T19:42:19.278133+0000","last_became_peered":"2024-07-28T19:42:19.278133+0000","last_unstale":"2024-07-28T19:49:14.427640+0000","last_undegraded":"2024-07-28T19:49:14.427640+0000","last_fullsized":"2024-07-28T19:49:14.427640+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:28:49.284657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":926,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.304360+0000","last_change":"2024-07-28T19:42:51.404654+0000","last_active":"2024-07-28T19:49:15.304360+0000","last_peered":"2024-07-28T19:49:15.304360+0000","last_clean":"2024-07-28T19:49:15.304360+0000","last_became_active":"2024-07-28T19:42:51.404064+0000","last_became_peered":"2024-07-28T19:42:51.404064+0000","last_unstale":"2024-07-28T19:49:15.304360+0000","last_undegraded":"2024-07-28T19:49:15.304360+0000","last_fullsized":"2024-07-28T19:49:15.304360+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:12:13.730589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":927,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.299575+0000","last_change":"2024-07-28T19:42:51.408176+0000","last_active":"2024-07-28T19:49:15.299575+0000","last_peered":"2024-07-28T19:49:15.299575+0000","last_clean":"2024-07-28T19:49:15.299575+0000","last_became_active":"2024-07-28T19:42:51.403879+0000","last_became_peered":"2024-07-28T19:42:51.403879+0000","last_unstale":"2024-07-28T19:49:15.299575+0000","last_undegraded":"2024-07-28T19:49:15.299575+0000","last_fullsized":"2024-07-28T19:49:15.299575+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:21:19.797025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":925,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.301376+0000","last_change":"2024-07-28T19:42:51.416379+0000","last_active":"2024-07-28T19:49:15.301376+0000","last_peered":"2024-07-28T19:49:15.301376+0000","last_clean":"2024-07-28T19:49:15.301376+0000","last_became_active":"2024-07-28T19:42:51.412668+0000","last_became_peered":"2024-07-28T19:42:51.412668+0000","last_unstale":"2024-07-28T19:49:15.301376+0000","last_undegraded":"2024-07-28T19:49:15.301376+0000","last_fullsized":"2024-07-28T19:49:15.301376+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:40:00.497141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":928,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.419000+0000","last_change":"2024-07-28T19:42:51.396048+0000","last_active":"2024-07-28T19:49:14.419000+0000","last_peered":"2024-07-28T19:49:14.419000+0000","last_clean":"2024-07-28T19:49:14.419000+0000","last_became_active":"2024-07-28T19:42:51.395490+0000","last_became_peered":"2024-07-28T19:42:51.395490+0000","last_unstale":"2024-07-28T19:49:14.419000+0000","last_undegraded":"2024-07-28T19:49:14.419000+0000","last_fullsized":"2024-07-28T19:49:14.419000+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:05:38.634689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":923,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.300285+0000","last_change":"2024-07-28T19:42:51.418278+0000","last_active":"2024-07-28T19:49:15.300285+0000","last_peered":"2024-07-28T19:49:15.300285+0000","last_clean":"2024-07-28T19:49:15.300285+0000","last_became_active":"2024-07-28T19:42:51.406218+0000","last_became_peered":"2024-07-28T19:42:51.406218+0000","last_unstale":"2024-07-28T19:49:15.300285+0000","last_undegraded":"2024-07-28T19:49:15.300285+0000","last_fullsized":"2024-07-28T19:49:15.300285+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:27:45.256338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":934,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.429207+0000","last_change":"2024-07-28T19:42:51.444620+0000","last_active":"2024-07-28T19:49:14.429207+0000","last_peered":"2024-07-28T19:49:14.429207+0000","last_clean":"2024-07-28T19:49:14.429207+0000","last_became_active":"2024-07-28T19:42:51.444220+0000","last_became_peered":"2024-07-28T19:42:51.444220+0000","last_unstale":"2024-07-28T19:49:14.429207+0000","last_undegraded":"2024-07-28T19:49:14.429207+0000","last_fullsized":"2024-07-28T19:49:14.429207+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:15:06.177140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":916,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.424919+0000","last_change":"2024-07-28T19:42:51.432640+0000","last_active":"2024-07-28T19:49:14.424919+0000","last_peered":"2024-07-28T19:49:14.424919+0000","last_clean":"2024-07-28T19:49:14.424919+0000","last_became_active":"2024-07-28T19:42:51.431525+0000","last_became_peered":"2024-07-28T19:42:51.431525+0000","last_unstale":"2024-07-28T19:49:14.424919+0000","last_undegraded":"2024-07-28T19:49:14.424919+0000","last_fullsized":"2024-07-28T19:49:14.424919+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:10:09.617690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":911,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.415777+0000","last_change":"2024-07-28T19:42:19.272179+0000","last_active":"2024-07-28T19:49:14.415777+0000","last_peered":"2024-07-28T19:49:14.415777+0000","last_clean":"2024-07-28T19:49:14.415777+0000","last_became_active":"2024-07-28T19:42:19.271578+0000","last_became_peered":"2024-07-28T19:42:19.271578+0000","last_unstale":"2024-07-28T19:49:14.415777+0000","last_undegraded":"2024-07-28T19:49:14.415777+0000","last_fullsized":"2024-07-28T19:49:14.415777+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:21:34.865305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":910,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.418727+0000","last_change":"2024-07-28T19:40:16.066388+0000","last_active":"2024-07-28T19:49:14.418727+0000","last_peered":"2024-07-28T19:49:14.418727+0000","last_clean":"2024-07-28T19:49:14.418727+0000","last_became_active":"2024-07-28T19:40:16.065619+0000","last_became_peered":"2024-07-28T19:40:16.065619+0000","last_unstale":"2024-07-28T19:49:14.418727+0000","last_undegraded":"2024-07-28T19:49:14.418727+0000","last_fullsized":"2024-07-28T19:49:14.418727+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:15:53.247278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":898,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.413337+0000","last_change":"2024-07-28T19:29:15.279653+0000","last_active":"2024-07-28T19:49:14.413337+0000","last_peered":"2024-07-28T19:49:14.413337+0000","last_clean":"2024-07-28T19:49:14.413337+0000","last_became_active":"2024-07-28T19:29:15.278829+0000","last_became_peered":"2024-07-28T19:29:15.278829+0000","last_unstale":"2024-07-28T19:49:14.413337+0000","last_undegraded":"2024-07-28T19:49:14.413337+0000","last_fullsized":"2024-07-28T19:49:14.413337+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:46:07.127995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":943,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.299793+0000","last_change":"2024-07-28T19:42:51.377351+0000","last_active":"2024-07-28T19:49:15.299793+0000","last_peered":"2024-07-28T19:49:15.299793+0000","last_clean":"2024-07-28T19:49:15.299793+0000","last_became_active":"2024-07-28T19:42:51.373120+0000","last_became_peered":"2024-07-28T19:42:51.373120+0000","last_unstale":"2024-07-28T19:49:15.299793+0000","last_undegraded":"2024-07-28T19:49:15.299793+0000","last_fullsized":"2024-07-28T19:49:15.299793+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:24:25.710095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":933,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.414772+0000","last_change":"2024-07-28T19:42:51.413620+0000","last_active":"2024-07-28T19:49:14.414772+0000","last_peered":"2024-07-28T19:49:14.414772+0000","last_clean":"2024-07-28T19:49:14.414772+0000","last_became_active":"2024-07-28T19:42:51.412048+0000","last_became_peered":"2024-07-28T19:42:51.412048+0000","last_unstale":"2024-07-28T19:49:14.414772+0000","last_undegraded":"2024-07-28T19:49:14.414772+0000","last_fullsized":"2024-07-28T19:49:14.414772+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:05:18.319837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":893,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411864+0000","last_change":"2024-07-28T19:29:16.445468+0000","last_active":"2024-07-28T19:49:14.411864+0000","last_peered":"2024-07-28T19:49:14.411864+0000","last_clean":"2024-07-28T19:49:14.411864+0000","last_became_active":"2024-07-28T19:29:16.444693+0000","last_became_peered":"2024-07-28T19:29:16.444693+0000","last_unstale":"2024-07-28T19:49:14.411864+0000","last_undegraded":"2024-07-28T19:49:14.411864+0000","last_fullsized":"2024-07-28T19:49:14.411864+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:13:39.927285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":932,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.415260+0000","last_change":"2024-07-28T19:42:51.411510+0000","last_active":"2024-07-28T19:49:14.415260+0000","last_peered":"2024-07-28T19:49:14.415260+0000","last_clean":"2024-07-28T19:49:14.415260+0000","last_became_active":"2024-07-28T19:42:51.411189+0000","last_became_peered":"2024-07-28T19:42:51.411189+0000","last_unstale":"2024-07-28T19:49:14.415260+0000","last_undegraded":"2024-07-28T19:49:14.415260+0000","last_fullsized":"2024-07-28T19:49:14.415260+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:27:19.712350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":923,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.418595+0000","last_change":"2024-07-28T19:39:41.991183+0000","last_active":"2024-07-28T19:49:14.418595+0000","last_peered":"2024-07-28T19:49:14.418595+0000","last_clean":"2024-07-28T19:49:14.418595+0000","last_became_active":"2024-07-28T19:39:41.990138+0000","last_became_peered":"2024-07-28T19:39:41.990138+0000","last_unstale":"2024-07-28T19:49:14.418595+0000","last_undegraded":"2024-07-28T19:49:14.418595+0000","last_fullsized":"2024-07-28T19:49:14.418595+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:59:07.444830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":908,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.293779+0000","last_change":"2024-07-28T19:42:19.284241+0000","last_active":"2024-07-28T19:49:15.293779+0000","last_peered":"2024-07-28T19:49:15.293779+0000","last_clean":"2024-07-28T19:49:15.293779+0000","last_became_active":"2024-07-28T19:42:19.281556+0000","last_became_peered":"2024-07-28T19:42:19.281556+0000","last_unstale":"2024-07-28T19:49:15.293779+0000","last_undegraded":"2024-07-28T19:49:15.293779+0000","last_fullsized":"2024-07-28T19:49:15.293779+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:02:57.369978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":913,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411991+0000","last_change":"2024-07-28T19:42:51.401979+0000","last_active":"2024-07-28T19:49:14.411991+0000","last_peered":"2024-07-28T19:49:14.411991+0000","last_clean":"2024-07-28T19:49:14.411991+0000","last_became_active":"2024-07-28T19:42:51.401314+0000","last_became_peered":"2024-07-28T19:42:51.401314+0000","last_unstale":"2024-07-28T19:49:14.411991+0000","last_undegraded":"2024-07-28T19:49:14.411991+0000","last_fullsized":"2024-07-28T19:49:14.411991+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:42:31.056894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":901,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.028696+0000","last_change":"2024-07-28T19:42:19.274756+0000","last_active":"2024-07-28T19:49:16.028696+0000","last_peered":"2024-07-28T19:49:16.028696+0000","last_clean":"2024-07-28T19:49:16.028696+0000","last_became_active":"2024-07-28T19:42:19.273800+0000","last_became_peered":"2024-07-28T19:42:19.273800+0000","last_unstale":"2024-07-28T19:49:16.028696+0000","last_undegraded":"2024-07-28T19:49:16.028696+0000","last_fullsized":"2024-07-28T19:49:16.028696+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:33:42.924762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":900,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.426414+0000","last_change":"2024-07-28T19:42:19.233491+0000","last_active":"2024-07-28T19:49:14.426414+0000","last_peered":"2024-07-28T19:49:14.426414+0000","last_clean":"2024-07-28T19:49:14.426414+0000","last_became_active":"2024-07-28T19:42:19.233074+0000","last_became_peered":"2024-07-28T19:42:19.233074+0000","last_unstale":"2024-07-28T19:49:14.426414+0000","last_undegraded":"2024-07-28T19:49:14.426414+0000","last_fullsized":"2024-07-28T19:49:14.426414+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:12:05.408285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":910,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.417079+0000","last_change":"2024-07-28T19:42:19.269055+0000","last_active":"2024-07-28T19:49:14.417079+0000","last_peered":"2024-07-28T19:49:14.417079+0000","last_clean":"2024-07-28T19:49:14.417079+0000","last_became_active":"2024-07-28T19:42:19.268658+0000","last_became_peered":"2024-07-28T19:42:19.268658+0000","last_unstale":"2024-07-28T19:49:14.417079+0000","last_undegraded":"2024-07-28T19:49:14.417079+0000","last_fullsized":"2024-07-28T19:49:14.417079+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:42:01.885987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":934,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.412795+0000","last_change":"2024-07-28T19:42:51.429453+0000","last_active":"2024-07-28T19:49:14.412795+0000","last_peered":"2024-07-28T19:49:14.412795+0000","last_clean":"2024-07-28T19:49:14.412795+0000","last_became_active":"2024-07-28T19:42:51.428701+0000","last_became_peered":"2024-07-28T19:42:51.428701+0000","last_unstale":"2024-07-28T19:49:14.412795+0000","last_undegraded":"2024-07-28T19:49:14.412795+0000","last_fullsized":"2024-07-28T19:49:14.412795+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:21:30.803400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":923,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.293558+0000","last_change":"2024-07-28T19:42:19.291360+0000","last_active":"2024-07-28T19:49:15.293558+0000","last_peered":"2024-07-28T19:49:15.293558+0000","last_clean":"2024-07-28T19:49:15.293558+0000","last_became_active":"2024-07-28T19:42:19.289985+0000","last_became_peered":"2024-07-28T19:42:19.289985+0000","last_unstale":"2024-07-28T19:49:15.293558+0000","last_undegraded":"2024-07-28T19:49:15.293558+0000","last_fullsized":"2024-07-28T19:49:15.293558+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:03:28.790331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5542,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5895,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201586,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308928,"kb_used_data":11576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93436224,"statfs":{"total":95995035648,"available":95678693376,"internally_reserved":0,"allocated":11853824,"data_stored":7760665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.1859999999999999,"15min":1.0620000000000001},"min":{"1min":0.42999999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.26,"5min":17.626000000000001,"15min":17.626000000000001},"last":1.23},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.4570000000000001,"15min":1.28},"min":{"1min":0.57299999999999995,"5min":0.375,"15min":0.375},"max":{"1min":1.2,"5min":29.181000000000001,"15min":29.181000000000001},"last":0.88200000000000001}]},{"osd":1,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.98099999999999998,"15min":0.98799999999999999},"min":{"1min":0.46400000000000002,"5min":0.45000000000000001,"15min":0.28299999999999997},"max":{"1min":1.151,"5min":12.973000000000001,"15min":14.095000000000001},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.2450000000000001,"15min":1.1679999999999999},"min":{"1min":0.53800000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.306,"5min":29.052,"15min":29.052},"last":0.70599999999999996}]},{"osd":2,"last update":"Sun Jul 28 19:49:59 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.411,"15min":1.2090000000000001},"min":{"1min":0.48799999999999999,"5min":0.41099999999999998,"15min":0.38600000000000001},"max":{"1min":1.3560000000000001,"5min":29.131,"15min":29.131},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.996,"15min":0.91900000000000004},"min":{"1min":0.47599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.2010000000000001,"5min":7.7300000000000004,"15min":7.7300000000000004},"last":1.0669999999999999}]},{"osd":3,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.0489999999999999,"15min":1.0129999999999999},"min":{"1min":0.441,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.177,"5min":7.5590000000000002,"15min":13.332000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.86799999999999999,"5min":1.115,"15min":1.4239999999999999},"min":{"1min":0.50700000000000001,"5min":0.495,"15min":0.495},"max":{"1min":1.1559999999999999,"5min":13.119,"15min":14.263},"last":1.1439999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:49:59 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.4630000000000001,"15min":1.143},"min":{"1min":0.55800000000000005,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.3480000000000001,"5min":29.547000000000001,"15min":29.547000000000001},"last":1.379},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.452,"15min":1.2330000000000001},"min":{"1min":0.57999999999999996,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2769999999999999,"5min":29.263000000000002,"15min":29.263000000000002},"last":1.0369999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.99299999999999999,"15min":1.1459999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.34000000000000002},"max":{"1min":1.284,"5min":7.657,"15min":33.914000000000001},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.014,"5min":1.6080000000000001,"15min":1.4039999999999999},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.376},"max":{"1min":1.1779999999999999,"5min":38.695,"15min":38.695},"last":1.28}]},{"osd":6,"last update":"Sun Jul 28 19:50:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.27,"15min":1.345},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.35299999999999998},"max":{"1min":1.3140000000000001,"5min":13.042999999999999,"15min":27.559000000000001},"last":1.3360000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.502,"15min":1.46},"min":{"1min":0.52400000000000002,"5min":0.41199999999999998,"15min":0.36799999999999999},"max":{"1min":1.218,"5min":29.218,"15min":31.202999999999999},"last":1.179}]}]},{"osd":6,"up_from":47,"seq":201863463228,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317240,"kb_used_data":11564,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93427912,"statfs":{"total":95995035648,"available":95670181888,"internally_reserved":0,"allocated":11841536,"data_stored":7760519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.94099999999999995,"15min":0.88500000000000001},"min":{"1min":0.55800000000000005,"5min":0.504,"15min":0.42999999999999999},"max":{"1min":1.3140000000000001,"5min":3.7069999999999999,"15min":3.7069999999999999},"last":1.117},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.95599999999999996,"15min":0.997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.35999999999999999},"max":{"1min":1.149,"5min":4.1950000000000003,"15min":16.055},"last":0.88}]},{"osd":1,"last update":"Sun Jul 28 19:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.98199999999999998,"15min":0.96999999999999997},"min":{"1min":0.497,"5min":0.497,"15min":0.377},"max":{"1min":1.204,"5min":4.4130000000000003,"15min":4.4130000000000003},"last":1.091},{"interface":"front","average":{"1min":0.996,"5min":1.0349999999999999,"15min":0.97899999999999998},"min":{"1min":0.53800000000000003,"5min":0.47199999999999998,"15min":0.378},"max":{"1min":1.4199999999999999,"5min":3.9079999999999999,"15min":3.9079999999999999},"last":0.93400000000000005}]},{"osd":2,"last update":"Sun Jul 28 19:50:09 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.0109999999999999,"15min":1.075},"min":{"1min":0.54500000000000004,"5min":0.437,"15min":0.437},"max":{"1min":1.2649999999999999,"5min":4.3579999999999997,"15min":4.4050000000000002},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.179,"15min":1.046},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2569999999999999,"5min":6.2380000000000004,"15min":6.2380000000000004},"last":0.97199999999999998}]},{"osd":3,"last update":"Sun Jul 28 19:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.097,"15min":1.0640000000000001},"min":{"1min":0.59299999999999997,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.242,"5min":5.7220000000000004,"15min":5.7220000000000004},"last":1},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.0960000000000001,"15min":1.1379999999999999},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.194,"5min":4.3029999999999999,"15min":4.6870000000000003},"last":1.0209999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:50:09 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.1659999999999999,"15min":1.03},"min":{"1min":0.48199999999999998,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.179,"5min":6.141,"15min":6.141},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.004,"5min":1.274,"15min":1.1950000000000001},"min":{"1min":0.67000000000000004,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.3100000000000001,"5min":5.8780000000000001,"15min":5.8780000000000001},"last":1.044}]},{"osd":5,"last update":"Sun Jul 28 19:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.117,"15min":1.048},"min":{"1min":0.51900000000000002,"5min":0.39000000000000001,"15min":0.316},"max":{"1min":1.155,"5min":6.5659999999999998,"15min":6.5659999999999998},"last":0.81200000000000006},{"interface":"front","average":{"1min":1.002,"5min":1.139,"15min":1.0860000000000001},"min":{"1min":0.45900000000000002,"5min":0.441,"15min":0.40400000000000003},"max":{"1min":1.226,"5min":5.9720000000000004,"15min":5.9720000000000004},"last":1.171}]},{"osd":7,"last update":"Sun Jul 28 19:50:21 2024","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.222,"15min":1.1479999999999999},"min":{"1min":0.68400000000000005,"5min":0.42099999999999999,"15min":0.33700000000000002},"max":{"1min":1.2190000000000001,"5min":6.3140000000000001,"15min":7.8620000000000001},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.052,"5min":1.222,"15min":1.181},"min":{"1min":0.58199999999999996,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.335,"5min":5.6299999999999999,"15min":8.6349999999999998},"last":1.2170000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724870,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308332,"kb_used_data":10976,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93436820,"statfs":{"total":95995035648,"available":95679303680,"internally_reserved":0,"allocated":11239424,"data_stored":7165965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.83499999999999996,"15min":0.89000000000000001},"min":{"1min":0.40999999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.9650000000000001,"5min":2.2400000000000002,"15min":13.148},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.96899999999999997,"15min":0.90500000000000003},"min":{"1min":0.50900000000000001,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":4.1859999999999999,"5min":6.1139999999999999,"15min":6.1139999999999999},"last":1.27}]},{"osd":1,"last update":"Sun Jul 28 19:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.014,"15min":1.044},"min":{"1min":0.59999999999999998,"5min":0.58299999999999996,"15min":0.314},"max":{"1min":1.208,"5min":2.4100000000000001,"15min":14.454000000000001},"last":0.879},{"interface":"front","average":{"1min":1.05,"5min":1.139,"15min":1.161},"min":{"1min":0.77800000000000002,"5min":0.54400000000000004,"15min":0.40100000000000002},"max":{"1min":1.3460000000000001,"5min":2.528,"15min":14.536},"last":1.1120000000000001}]},{"osd":2,"last update":"Sun Jul 28 19:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.92300000000000004,"15min":1.22},"min":{"1min":0.61799999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":3.75,"5min":3.75,"15min":12.542},"last":1.139},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0449999999999999,"15min":1.3620000000000001},"min":{"1min":0.58799999999999997,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":3.8239999999999998,"5min":3.8239999999999998,"15min":12.727},"last":1.2290000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.048,"15min":0.95799999999999996},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.41499999999999998},"max":{"1min":1.224,"5min":6.1980000000000004,"15min":6.1980000000000004},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.147,"15min":1.0780000000000001},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.47599999999999998},"max":{"1min":1.216,"5min":2.6469999999999998,"15min":2.6469999999999998},"last":0.99099999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:50:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.073,"15min":1.101},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":2.3410000000000002,"5min":5.5250000000000004,"15min":5.5250000000000004},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.161,"5min":1.1859999999999999,"15min":1.079},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":3.915,"5min":6.9660000000000002,"15min":6.9660000000000002},"last":1.0680000000000001}]},{"osd":6,"last update":"Sun Jul 28 19:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.0189999999999999,"15min":1.0529999999999999},"min":{"1min":0.42599999999999999,"5min":0.39200000000000002,"15min":0.33700000000000002},"max":{"1min":1.1850000000000001,"5min":5.2489999999999997,"15min":12.981999999999999},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.24,"15min":1.113},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.42699999999999999},"max":{"1min":1.381,"5min":7.0449999999999999,"15min":7.0449999999999999},"last":1.3260000000000001}]},{"osd":7,"last update":"Sun Jul 28 19:50:21 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.173,"15min":1.105},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.36899999999999999},"max":{"1min":1.419,"5min":7.1070000000000002,"15min":9.6110000000000007},"last":1.175},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.216,"15min":1.1859999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.456,"5min":5.6230000000000002,"15min":15.936},"last":0.91600000000000004}]}]},{"osd":4,"up_from":652,"seq":2800318677088,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":64952,"kb_used_data":10992,"kb_used_omap":67,"kb_used_meta":53884,"kb_avail":93680200,"statfs":{"total":95995035648,"available":95928524800,"internally_reserved":0,"allocated":11255808,"data_stored":7170227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69313,"internal_metadata":55177535},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.3169999999999999,"15min":0.98499999999999999},"min":{"1min":0.65900000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.827,"5min":20.173999999999999,"15min":20.173999999999999},"last":0.873},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.071,"15min":0.93300000000000005},"min":{"1min":0.64300000000000002,"5min":0.501,"15min":0.42599999999999999},"max":{"1min":1.8879999999999999,"5min":4.1109999999999998,"15min":4.1109999999999998},"last":1.0820000000000001}]},{"osd":1,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.996,"15min":0.97199999999999998},"min":{"1min":0.628,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.2010000000000001,"5min":2.3519999999999999,"15min":2.3519999999999999},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.96999999999999997,"15min":1.0249999999999999},"min":{"1min":0.51700000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.274,"5min":1.4399999999999999,"15min":2.1259999999999999},"last":1.1040000000000001}]},{"osd":2,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.0369999999999999,"15min":1.042},"min":{"1min":0.68600000000000005,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.3180000000000001,"5min":2.5019999999999998,"15min":2.5019999999999998},"last":0.996},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.1639999999999999,"15min":1.056},"min":{"1min":0.66800000000000004,"5min":0.504,"15min":0.504},"max":{"1min":1.2829999999999999,"5min":17.503,"15min":17.503},"last":0.94999999999999996}]},{"osd":3,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.042,"15min":1.004},"min":{"1min":0.75900000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.242,"5min":2.2200000000000002,"15min":2.2200000000000002},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.077,"5min":1.101,"15min":1.1639999999999999},"min":{"1min":0.93000000000000005,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.2230000000000001,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":1.155}]},{"osd":5,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0329999999999999,"15min":1.0640000000000001},"min":{"1min":0.77300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2490000000000001,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":1.2050000000000001},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.1870000000000001,"15min":1.1379999999999999},"min":{"1min":0.60599999999999998,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.373,"5min":17.393999999999998,"15min":17.393999999999998},"last":0.77500000000000002}]},{"osd":6,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.236,"15min":1.1639999999999999},"min":{"1min":0.66600000000000004,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3979999999999999,"5min":17.664000000000001,"15min":17.664000000000001},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.1879999999999999,"15min":1.141},"min":{"1min":0.76500000000000001,"5min":0.55300000000000005,"15min":0.47899999999999998},"max":{"1min":1.363,"5min":7.2220000000000004,"15min":7.2220000000000004},"last":0.72899999999999998}]},{"osd":7,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.1619999999999999,"15min":1.1799999999999999},"min":{"1min":0.73299999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3480000000000001,"5min":4.0170000000000003,"15min":4.0170000000000003},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.2210000000000001,"15min":1.3320000000000001},"min":{"1min":0.89600000000000002,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.4330000000000001,"5min":4.7640000000000002,"15min":4.7640000000000002},"last":0.82899999999999996}]}]},{"osd":3,"up_from":643,"seq":2761663971430,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":92856,"kb_used_data":10972,"kb_used_omap":69,"kb_used_meta":57914,"kb_avail":93652296,"statfs":{"total":95995035648,"available":95899951104,"internally_reserved":0,"allocated":11235328,"data_stored":7165415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70822,"internal_metadata":59304794},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.745,"15min":0.65900000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":0.98399999999999999,"5min":4.0519999999999996,"15min":4.0519999999999996},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.746,"5min":0.88100000000000001,"15min":0.73599999999999999},"min":{"1min":0.44,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.0860000000000001,"5min":4.7009999999999996,"15min":6.4370000000000003},"last":0.82899999999999996}]},{"osd":1,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.85099999999999998,"15min":0.80300000000000005},"min":{"1min":0.41599999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.242,"5min":3.177,"15min":3.294},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.88800000000000001,"15min":0.80700000000000005},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":1.038,"5min":8.7010000000000005,"15min":20.555},"last":0.77500000000000002}]},{"osd":2,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.89700000000000002,"15min":0.84099999999999997},"min":{"1min":0.36099999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.169,"5min":5.5250000000000004,"15min":5.5250000000000004},"last":0.755},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.90300000000000002,"15min":0.88500000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.29399999999999998},"max":{"1min":1.091,"5min":3.9220000000000002,"15min":20.242000000000001},"last":0.91700000000000004}]},{"osd":4,"last update":"Sun Jul 28 19:50:08 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0429999999999999,"15min":0.98799999999999999},"min":{"1min":0.80900000000000005,"5min":0.501,"15min":0.437},"max":{"1min":1.4239999999999999,"5min":3.5179999999999998,"15min":3.831},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.026,"5min":1.046,"15min":1.0640000000000001},"min":{"1min":0.72299999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.4119999999999999,"5min":3.3159999999999998,"15min":20.440000000000001},"last":0.99399999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.0620000000000001,"15min":1.04},"min":{"1min":0.755,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2310000000000001,"5min":8.8109999999999999,"15min":23.391999999999999},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.1040000000000001,"15min":1.036},"min":{"1min":0.65900000000000003,"5min":0.51700000000000002,"15min":0.39400000000000002},"max":{"1min":1.415,"5min":3.4529999999999998,"15min":3.4529999999999998},"last":0.94299999999999995}]},{"osd":6,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.2929999999999999,"15min":1.1379999999999999},"min":{"1min":0.629,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":4.1040000000000001,"5min":11.532999999999999,"15min":11.532999999999999},"last":0.875},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1040000000000001,"15min":1.131},"min":{"1min":0.86599999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.5920000000000001,"5min":2.3079999999999998,"15min":20.358000000000001},"last":1.0469999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.169,"15min":1.137},"min":{"1min":0.70599999999999996,"5min":0.442,"15min":0.442},"max":{"1min":1.341,"5min":9.0700000000000003,"15min":21.030000000000001},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.208,"15min":1.153},"min":{"1min":0.68300000000000005,"5min":0.45900000000000002,"15min":0.41399999999999998},"max":{"1min":4.1929999999999996,"5min":4.1929999999999996,"15min":4.1929999999999996},"last":1.024}]}]},{"osd":2,"up_from":628,"seq":2697239461996,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":97172,"kb_used_data":10976,"kb_used_omap":72,"kb_used_meta":58103,"kb_avail":93647980,"statfs":{"total":95995035648,"available":95895531520,"internally_reserved":0,"allocated":11239424,"data_stored":7165929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74488,"internal_metadata":59497736},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.68000000000000005,"15min":0.59399999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.21099999999999999},"max":{"1min":0.98999999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.749,"5min":0.83899999999999997,"15min":0.70299999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":1.143,"5min":4.5389999999999997,"15min":4.5389999999999997},"last":0.90900000000000003}]},{"osd":1,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":1.018,"15min":0.76400000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.30599999999999999},"max":{"1min":1.038,"5min":14.276999999999999,"15min":14.276999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.73099999999999998,"5min":1.1299999999999999,"15min":0.76700000000000002},"min":{"1min":0.36799999999999999,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":1.044,"5min":25.533000000000001,"15min":25.533000000000001},"last":0.86599999999999999}]},{"osd":3,"last update":"Sun Jul 28 19:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.88300000000000001,"15min":0.80700000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.35099999999999998},"max":{"1min":1.087,"5min":2.754,"15min":2.754},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.877,"15min":0.80500000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.27700000000000002},"max":{"1min":1.0780000000000001,"5min":3.0179999999999998,"15min":3.0179999999999998},"last":0.95599999999999996}]},{"osd":4,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.1180000000000001,"15min":1.2150000000000001},"min":{"1min":0.76300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.2629999999999999,"5min":4.2699999999999996,"15min":5.2990000000000004},"last":1.008},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.97299999999999998,"15min":1.1319999999999999},"min":{"1min":0.69099999999999995,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1200000000000001,"5min":1.6100000000000001,"15min":5.3390000000000004},"last":1.1639999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.0740000000000001,"15min":1.0980000000000001},"min":{"1min":0.64100000000000001,"5min":0.42199999999999999,"15min":0.32600000000000001},"max":{"1min":1.276,"5min":2.4399999999999999,"15min":8.2460000000000004},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.083,"15min":1.1100000000000001},"min":{"1min":0.82899999999999996,"5min":0.68000000000000005,"15min":0.40100000000000002},"max":{"1min":1.4370000000000001,"5min":2.2120000000000002,"15min":5.1440000000000001},"last":1.1499999999999999}]},{"osd":6,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.1180000000000001,"15min":1.0900000000000001},"min":{"1min":0.73399999999999999,"5min":0.48199999999999998,"15min":0.41399999999999998},"max":{"1min":1.3740000000000001,"5min":2.907,"15min":5.7599999999999998},"last":1.119},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0680000000000001,"15min":1.1499999999999999},"min":{"1min":0.61699999999999999,"5min":0.48999999999999999,"15min":0.36899999999999999},"max":{"1min":1.3380000000000001,"5min":2.657,"15min":8.2119999999999997},"last":1.0720000000000001}]},{"osd":7,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0900000000000001,"15min":1.1879999999999999},"min":{"1min":0.68799999999999994,"5min":0.44,"15min":0.44},"max":{"1min":1.278,"5min":1.7070000000000001,"15min":5.3860000000000001},"last":1.242},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.1399999999999999,"15min":1.155},"min":{"1min":0.71599999999999997,"5min":0.71199999999999997,"15min":0.46800000000000003},"max":{"1min":1.3740000000000001,"5min":2.1160000000000001,"15min":4.0579999999999998},"last":1.0309999999999999}]}]},{"osd":1,"up_from":540,"seq":2319282339967,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":60712,"kb_used_data":10968,"kb_used_omap":48,"kb_used_meta":49679,"kb_avail":93684440,"statfs":{"total":95995035648,"available":95932866560,"internally_reserved":0,"allocated":11231232,"data_stored":7165325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49798,"internal_metadata":50871674},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.73399999999999999,"15min":0.72199999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.247},"max":{"1min":0.90700000000000003,"5min":2.7269999999999999,"15min":2.7269999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.75600000000000001,"15min":0.71299999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":1.2030000000000001,"5min":4.1909999999999998,"15min":4.1909999999999998},"last":0.79800000000000004}]},{"osd":2,"last update":"Sun Jul 28 19:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.214,"15min":0.88600000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.36499999999999999},"max":{"1min":1.218,"5min":15.945,"15min":15.945},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.107,"15min":0.89900000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.0580000000000001,"5min":15.833,"15min":15.833},"last":1.0109999999999999}]},{"osd":3,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.91800000000000004,"15min":0.84399999999999997},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.246},"max":{"1min":1.0640000000000001,"5min":9.4169999999999998,"15min":9.4169999999999998},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.78600000000000003,"15min":0.88400000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.96799999999999997,"5min":2.0289999999999999,"15min":11.183},"last":0.85499999999999998}]},{"osd":4,"last update":"Sun Jul 28 19:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.1140000000000001,"15min":1.01},"min":{"1min":0.59999999999999998,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.2829999999999999,"5min":10.903,"15min":10.903},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.1299999999999999,"15min":1.1399999999999999},"min":{"1min":0.65400000000000003,"5min":0.379,"15min":0.379},"max":{"1min":1.238,"5min":11.984999999999999,"15min":18.853000000000002},"last":1.1619999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.087,"15min":1.0429999999999999},"min":{"1min":0.56799999999999995,"5min":0.54700000000000004,"15min":0.42899999999999999},"max":{"1min":1.3480000000000001,"5min":4.4640000000000004,"15min":8.2059999999999995},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.1240000000000001,"15min":1.044},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.42099999999999999},"max":{"1min":1.1799999999999999,"5min":9.3360000000000003,"15min":9.3360000000000003},"last":1.089}]},{"osd":6,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.069,"15min":1.0900000000000001},"min":{"1min":0.60999999999999999,"5min":0.497,"15min":0.497},"max":{"1min":1.268,"5min":3.8250000000000002,"15min":3.8250000000000002},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.1379999999999999,"15min":1.095},"min":{"1min":0.61699999999999999,"5min":0.56499999999999995,"15min":0.50700000000000001},"max":{"1min":1.425,"5min":4.3689999999999998,"15min":4.3689999999999998},"last":1.2150000000000001}]},{"osd":7,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.1060000000000001,"15min":1.129},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.38800000000000001},"max":{"1min":1.2490000000000001,"5min":1.671,"15min":9.0679999999999996},"last":1.2490000000000001},{"interface":"front","average":{"1min":1.018,"5min":1.1160000000000001,"15min":1.194},"min":{"1min":0.66000000000000003,"5min":0.58099999999999996,"15min":0.52300000000000002},"max":{"1min":1.294,"5min":2.1019999999999999,"15min":11},"last":1.1890000000000001}]}]},{"osd":0,"up_from":506,"seq":2173253451909,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65452,"kb_used_data":11556,"kb_used_omap":55,"kb_used_meta":53832,"kb_avail":93679700,"statfs":{"total":95995035648,"available":95928012800,"internally_reserved":0,"allocated":11833344,"data_stored":7756285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56449,"internal_metadata":55124863},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Jul 28 19:49:33 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.64300000000000002,"15min":0.83299999999999996},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.22600000000000001},"max":{"1min":0.94399999999999995,"5min":1.1359999999999999,"15min":9.5730000000000004},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.75900000000000001,"15min":0.74399999999999999},"min":{"1min":0.42599999999999999,"5min":0.34599999999999997,"15min":0.33800000000000002},"max":{"1min":0.99399999999999999,"5min":1.78,"15min":1.843},"last":0.74099999999999999}]},{"osd":2,"last update":"Sun Jul 28 19:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.76200000000000001,"15min":0.68500000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.24199999999999999},"max":{"1min":1.0289999999999999,"5min":1.8779999999999999,"15min":1.8779999999999999},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.83999999999999997,"15min":0.69399999999999995},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29399999999999998},"max":{"1min":2.3090000000000002,"5min":8.3330000000000002,"15min":8.3330000000000002},"last":0.69299999999999995}]},{"osd":3,"last update":"Sun Jul 28 19:49:33 2024","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":1.077,"15min":0.88400000000000001},"min":{"1min":0.35299999999999998,"5min":0.32600000000000001,"15min":0.28899999999999998},"max":{"1min":2.1339999999999999,"5min":14.456,"15min":14.456},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.4279999999999999,"15min":1.0209999999999999},"min":{"1min":0.38500000000000001,"5min":0.36299999999999999,"15min":0.32900000000000001},"max":{"1min":2.2109999999999999,"5min":27.617999999999999,"15min":27.617999999999999},"last":0.84999999999999998}]},{"osd":4,"last update":"Sun Jul 28 19:50:00 2024","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.4510000000000001,"15min":1.242},"min":{"1min":0.68899999999999995,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":2.3519999999999999,"5min":27.690999999999999,"15min":27.690999999999999},"last":1.1890000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1,"15min":0.96899999999999997},"min":{"1min":0.66900000000000004,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3680000000000001,"5min":2.2919999999999998,"15min":2.2919999999999998},"last":1.1020000000000001}]},{"osd":5,"last update":"Sun Jul 28 19:50:00 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.96599999999999997,"15min":0.98099999999999998},"min":{"1min":0.50600000000000001,"5min":0.44800000000000001,"15min":0.35599999999999998},"max":{"1min":1.71,"5min":4.9619999999999997,"15min":8.8179999999999996},"last":0.99099999999999999},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.0589999999999999,"15min":1.0329999999999999},"min":{"1min":0.64300000000000002,"5min":0.57999999999999996,"15min":0.38800000000000001},"max":{"1min":1.4490000000000001,"5min":2.3980000000000001,"15min":2.3980000000000001},"last":1.02}]},{"osd":6,"last update":"Sun Jul 28 19:50:00 2024","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.071,"15min":1.0600000000000001},"min":{"1min":0.621,"5min":0.56000000000000005,"15min":0.41499999999999998},"max":{"1min":1.5920000000000001,"5min":2.4750000000000001,"15min":2.4750000000000001},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.093,"5min":1.6419999999999999,"15min":1.2749999999999999},"min":{"1min":0.68799999999999994,"5min":0.434,"15min":0.42499999999999999},"max":{"1min":2.266,"5min":27.832000000000001,"15min":27.832000000000001},"last":1.054}]},{"osd":7,"last update":"Sun Jul 28 19:50:00 2024","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.488,"15min":1.248},"min":{"1min":0.55400000000000005,"5min":0.49399999999999999,"15min":0.47099999999999997},"max":{"1min":2.4020000000000001,"5min":27.888999999999999,"15min":27.888999999999999},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.05,"15min":1.0820000000000001},"min":{"1min":0.58099999999999996,"5min":0.56999999999999995,"15min":0.44},"max":{"1min":1.784,"5min":2.2080000000000002,"15min":9.3759999999999994},"last":0.89000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1355,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4540,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-28T19:50:41.514 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph pg dump --format=json 2024-07-28T19:50:41.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:41 smithi138 bash[67894]: cluster 2024-07-28T19:50:40.108971+0000 mgr.y (mgr.25358) 542 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:42.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:42 smithi138 bash[67894]: audit 2024-07-28T19:50:40.760827+0000 mgr.y (mgr.25358) 543 : audit [DBG] from='client.44642 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:50:42.557 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:50:42 smithi138 bash[60615]: ts=2024-07-28T19:50:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:50:42.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:42 smithi100 bash[80199]: audit 2024-07-28T19:50:40.760827+0000 mgr.y (mgr.25358) 543 : audit [DBG] from='client.44642 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:50:42.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:42 smithi100 bash[82644]: audit 2024-07-28T19:50:40.760827+0000 mgr.y (mgr.25358) 543 : audit [DBG] from='client.44642 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:50:43.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:50:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:50:42] "GET /metrics HTTP/1.1" 200 34267 "" "Prometheus/2.43.0" 2024-07-28T19:50:43.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:43 smithi138 bash[67894]: cluster 2024-07-28T19:50:42.109725+0000 mgr.y (mgr.25358) 544 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:43.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:43 smithi100 bash[80199]: cluster 2024-07-28T19:50:42.109725+0000 mgr.y (mgr.25358) 544 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:43.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:43 smithi100 bash[82644]: cluster 2024-07-28T19:50:42.109725+0000 mgr.y (mgr.25358) 544 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:45.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:45 smithi138 bash[67894]: cluster 2024-07-28T19:50:44.110670+0000 mgr.y (mgr.25358) 545 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:45.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:45 smithi100 bash[80199]: cluster 2024-07-28T19:50:44.110670+0000 mgr.y (mgr.25358) 545 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:45.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:45 smithi100 bash[82644]: cluster 2024-07-28T19:50:44.110670+0000 mgr.y (mgr.25358) 545 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:46.290 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:47.526 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:47 smithi100 bash[80199]: cluster 2024-07-28T19:50:46.111573+0000 mgr.y (mgr.25358) 546 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:47.527 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:47 smithi100 bash[82644]: cluster 2024-07-28T19:50:46.111573+0000 mgr.y (mgr.25358) 546 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:47.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:47 smithi138 bash[67894]: cluster 2024-07-28T19:50:46.111573+0000 mgr.y (mgr.25358) 546 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:49.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:49 smithi100 bash[82644]: cluster 2024-07-28T19:50:48.112320+0000 mgr.y (mgr.25358) 547 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:49.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:49 smithi100 bash[80199]: cluster 2024-07-28T19:50:48.112320+0000 mgr.y (mgr.25358) 547 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:49.547 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:50:49.549 INFO:teuthology.orchestra.run.smithi100.stderr:dumped all 2024-07-28T19:50:49.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:49 smithi138 bash[67894]: cluster 2024-07-28T19:50:48.112320+0000 mgr.y (mgr.25358) 547 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:50.259 INFO:teuthology.orchestra.run.smithi100.stdout:{"pg_ready":true,"pg_map":{"version":1116,"stamp":"2024-07-28T19:50:48.111873+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595910,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":538,"num_read_kb":964,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1315644,"kb_used_data":89580,"kb_used_omap":318,"kb_used_meta":1173761,"kb_avail":748645572,"statfs":{"total":767960285184,"available":766613065728,"internally_reserved":0,"allocated":91729920,"data_stored":59110330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":325982,"internal_metadata":1201931938},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005180"},"pg_stats":[{"pgid":"3.1f","version":"70'5","reported_seq":912,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.030379+0000","last_change":"2024-07-28T19:41:50.992404+0000","last_active":"2024-07-28T19:49:16.030379+0000","last_peered":"2024-07-28T19:49:16.030379+0000","last_clean":"2024-07-28T19:49:16.030379+0000","last_became_active":"2024-07-28T19:41:50.991667+0000","last_became_peered":"2024-07-28T19:41:50.991667+0000","last_unstale":"2024-07-28T19:49:16.030379+0000","last_undegraded":"2024-07-28T19:49:16.030379+0000","last_fullsized":"2024-07-28T19:49:16.030379+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:43:38.150311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":926,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.298309+0000","last_change":"2024-07-28T19:42:51.406228+0000","last_active":"2024-07-28T19:49:15.298309+0000","last_peered":"2024-07-28T19:49:15.298309+0000","last_clean":"2024-07-28T19:49:15.298309+0000","last_became_active":"2024-07-28T19:42:51.398223+0000","last_became_peered":"2024-07-28T19:42:51.398223+0000","last_unstale":"2024-07-28T19:49:15.298309+0000","last_undegraded":"2024-07-28T19:49:15.298309+0000","last_fullsized":"2024-07-28T19:49:15.298309+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:21:35.052391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":917,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.293766+0000","last_change":"2024-07-28T19:42:19.269168+0000","last_active":"2024-07-28T19:49:15.293766+0000","last_peered":"2024-07-28T19:49:15.293766+0000","last_clean":"2024-07-28T19:49:15.293766+0000","last_became_active":"2024-07-28T19:42:19.268312+0000","last_became_peered":"2024-07-28T19:42:19.268312+0000","last_unstale":"2024-07-28T19:49:15.293766+0000","last_undegraded":"2024-07-28T19:49:15.293766+0000","last_fullsized":"2024-07-28T19:49:15.293766+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:50:25.364241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":930,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.291650+0000","last_change":"2024-07-28T19:42:19.243100+0000","last_active":"2024-07-28T19:49:15.291650+0000","last_peered":"2024-07-28T19:49:15.291650+0000","last_clean":"2024-07-28T19:49:15.291650+0000","last_became_active":"2024-07-28T19:42:19.242831+0000","last_became_peered":"2024-07-28T19:42:19.242831+0000","last_unstale":"2024-07-28T19:49:15.291650+0000","last_undegraded":"2024-07-28T19:49:15.291650+0000","last_fullsized":"2024-07-28T19:49:15.291650+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:09:12.249965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":914,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.416932+0000","last_change":"2024-07-28T19:42:51.419502+0000","last_active":"2024-07-28T19:49:14.416932+0000","last_peered":"2024-07-28T19:49:14.416932+0000","last_clean":"2024-07-28T19:49:14.416932+0000","last_became_active":"2024-07-28T19:42:51.419110+0000","last_became_peered":"2024-07-28T19:42:51.419110+0000","last_unstale":"2024-07-28T19:49:14.416932+0000","last_undegraded":"2024-07-28T19:49:14.416932+0000","last_fullsized":"2024-07-28T19:49:14.416932+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:24:21.005730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":940,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.300018+0000","last_change":"2024-07-28T19:42:51.401545+0000","last_active":"2024-07-28T19:49:15.300018+0000","last_peered":"2024-07-28T19:49:15.300018+0000","last_clean":"2024-07-28T19:49:15.300018+0000","last_became_active":"2024-07-28T19:42:51.400879+0000","last_became_peered":"2024-07-28T19:42:51.400879+0000","last_unstale":"2024-07-28T19:49:15.300018+0000","last_undegraded":"2024-07-28T19:49:15.300018+0000","last_fullsized":"2024-07-28T19:49:15.300018+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:22:22.537822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":931,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.416084+0000","last_change":"2024-07-28T19:42:51.421716+0000","last_active":"2024-07-28T19:49:14.416084+0000","last_peered":"2024-07-28T19:49:14.416084+0000","last_clean":"2024-07-28T19:49:14.416084+0000","last_became_active":"2024-07-28T19:42:51.421372+0000","last_became_peered":"2024-07-28T19:42:51.421372+0000","last_unstale":"2024-07-28T19:49:14.416084+0000","last_undegraded":"2024-07-28T19:49:14.416084+0000","last_fullsized":"2024-07-28T19:49:14.416084+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:29:04.117086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":940,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.303431+0000","last_change":"2024-07-28T19:42:51.425099+0000","last_active":"2024-07-28T19:49:15.303431+0000","last_peered":"2024-07-28T19:49:15.303431+0000","last_clean":"2024-07-28T19:49:15.303431+0000","last_became_active":"2024-07-28T19:42:51.420828+0000","last_became_peered":"2024-07-28T19:42:51.420828+0000","last_unstale":"2024-07-28T19:49:15.303431+0000","last_undegraded":"2024-07-28T19:49:15.303431+0000","last_fullsized":"2024-07-28T19:49:15.303431+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:43:09.431596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":906,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.027218+0000","last_change":"2024-07-28T19:42:51.433065+0000","last_active":"2024-07-28T19:49:16.027218+0000","last_peered":"2024-07-28T19:49:16.027218+0000","last_clean":"2024-07-28T19:49:16.027218+0000","last_became_active":"2024-07-28T19:42:51.432308+0000","last_became_peered":"2024-07-28T19:42:51.432308+0000","last_unstale":"2024-07-28T19:49:16.027218+0000","last_undegraded":"2024-07-28T19:49:16.027218+0000","last_fullsized":"2024-07-28T19:49:16.027218+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:54:54.488258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":896,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.689335+0000","last_change":"2024-07-28T19:42:19.241821+0000","last_active":"2024-07-28T19:49:15.689335+0000","last_peered":"2024-07-28T19:49:15.689335+0000","last_clean":"2024-07-28T19:49:15.689335+0000","last_became_active":"2024-07-28T19:42:19.241459+0000","last_became_peered":"2024-07-28T19:42:19.241459+0000","last_unstale":"2024-07-28T19:49:15.689335+0000","last_undegraded":"2024-07-28T19:49:15.689335+0000","last_fullsized":"2024-07-28T19:49:15.689335+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:58:56.626908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":946,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.300501+0000","last_change":"2024-07-28T19:42:51.408616+0000","last_active":"2024-07-28T19:49:15.300501+0000","last_peered":"2024-07-28T19:49:15.300501+0000","last_clean":"2024-07-28T19:49:15.300501+0000","last_became_active":"2024-07-28T19:42:51.405947+0000","last_became_peered":"2024-07-28T19:42:51.405947+0000","last_unstale":"2024-07-28T19:49:15.300501+0000","last_undegraded":"2024-07-28T19:49:15.300501+0000","last_fullsized":"2024-07-28T19:49:15.300501+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:53:22.106233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":923,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.294393+0000","last_change":"2024-07-28T19:42:51.441869+0000","last_active":"2024-07-28T19:49:15.294393+0000","last_peered":"2024-07-28T19:49:15.294393+0000","last_clean":"2024-07-28T19:49:15.294393+0000","last_became_active":"2024-07-28T19:42:51.441528+0000","last_became_peered":"2024-07-28T19:42:51.441528+0000","last_unstale":"2024-07-28T19:49:15.294393+0000","last_undegraded":"2024-07-28T19:49:15.294393+0000","last_fullsized":"2024-07-28T19:49:15.294393+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:27:32.962528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":921,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.733890+0000","last_change":"2024-07-28T19:42:51.381104+0000","last_active":"2024-07-28T19:49:15.733890+0000","last_peered":"2024-07-28T19:49:15.733890+0000","last_clean":"2024-07-28T19:49:15.733890+0000","last_became_active":"2024-07-28T19:42:51.380818+0000","last_became_peered":"2024-07-28T19:42:51.380818+0000","last_unstale":"2024-07-28T19:49:15.733890+0000","last_undegraded":"2024-07-28T19:49:15.733890+0000","last_fullsized":"2024-07-28T19:49:15.733890+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:09:25.890587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":902,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.028523+0000","last_change":"2024-07-28T19:42:19.274570+0000","last_active":"2024-07-28T19:49:16.028523+0000","last_peered":"2024-07-28T19:49:16.028523+0000","last_clean":"2024-07-28T19:49:16.028523+0000","last_became_active":"2024-07-28T19:42:19.273406+0000","last_became_peered":"2024-07-28T19:42:19.273406+0000","last_unstale":"2024-07-28T19:49:16.028523+0000","last_undegraded":"2024-07-28T19:49:16.028523+0000","last_fullsized":"2024-07-28T19:49:16.028523+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:07:20.283774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":926,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.291348+0000","last_change":"2024-07-28T19:42:19.241728+0000","last_active":"2024-07-28T19:49:15.291348+0000","last_peered":"2024-07-28T19:49:15.291348+0000","last_clean":"2024-07-28T19:49:15.291348+0000","last_became_active":"2024-07-28T19:42:19.241448+0000","last_became_peered":"2024-07-28T19:42:19.241448+0000","last_unstale":"2024-07-28T19:49:15.291348+0000","last_undegraded":"2024-07-28T19:49:15.291348+0000","last_fullsized":"2024-07-28T19:49:15.291348+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:01:23.574357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":910,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.419553+0000","last_change":"2024-07-28T19:40:16.070038+0000","last_active":"2024-07-28T19:49:14.419553+0000","last_peered":"2024-07-28T19:49:14.419553+0000","last_clean":"2024-07-28T19:49:14.419553+0000","last_became_active":"2024-07-28T19:40:16.069305+0000","last_became_peered":"2024-07-28T19:40:16.069305+0000","last_unstale":"2024-07-28T19:49:14.419553+0000","last_undegraded":"2024-07-28T19:49:14.419553+0000","last_fullsized":"2024-07-28T19:49:14.419553+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T06:20:30.019331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":908,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.291797+0000","last_change":"2024-07-28T19:42:19.266717+0000","last_active":"2024-07-28T19:49:15.291797+0000","last_peered":"2024-07-28T19:49:15.291797+0000","last_clean":"2024-07-28T19:49:15.291797+0000","last_became_active":"2024-07-28T19:42:19.266351+0000","last_became_peered":"2024-07-28T19:42:19.266351+0000","last_unstale":"2024-07-28T19:49:15.291797+0000","last_undegraded":"2024-07-28T19:49:15.291797+0000","last_fullsized":"2024-07-28T19:49:15.291797+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T06:56:16.625754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":900,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.027248+0000","last_change":"2024-07-28T19:42:19.263249+0000","last_active":"2024-07-28T19:49:16.027248+0000","last_peered":"2024-07-28T19:49:16.027248+0000","last_clean":"2024-07-28T19:49:16.027248+0000","last_became_active":"2024-07-28T19:42:19.262846+0000","last_became_peered":"2024-07-28T19:42:19.262846+0000","last_unstale":"2024-07-28T19:49:16.027248+0000","last_undegraded":"2024-07-28T19:49:16.027248+0000","last_fullsized":"2024-07-28T19:49:16.027248+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:56:59.694417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":869,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.291845+0000","last_change":"2024-07-28T19:42:19.230447+0000","last_active":"2024-07-28T19:49:15.291845+0000","last_peered":"2024-07-28T19:49:15.291845+0000","last_clean":"2024-07-28T19:49:15.291845+0000","last_became_active":"2024-07-28T19:42:19.229707+0000","last_became_peered":"2024-07-28T19:42:19.229707+0000","last_unstale":"2024-07-28T19:49:15.291845+0000","last_undegraded":"2024-07-28T19:49:15.291845+0000","last_fullsized":"2024-07-28T19:49:15.291845+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:03:22.965514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":911,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.416623+0000","last_change":"2024-07-28T19:40:16.062070+0000","last_active":"2024-07-28T19:49:14.416623+0000","last_peered":"2024-07-28T19:49:14.416623+0000","last_clean":"2024-07-28T19:49:14.416623+0000","last_became_active":"2024-07-28T19:40:16.061253+0000","last_became_peered":"2024-07-28T19:40:16.061253+0000","last_unstale":"2024-07-28T19:49:14.416623+0000","last_undegraded":"2024-07-28T19:49:14.416623+0000","last_fullsized":"2024-07-28T19:49:14.416623+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:09:42.378599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":935,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411877+0000","last_change":"2024-07-28T19:42:51.425444+0000","last_active":"2024-07-28T19:49:14.411877+0000","last_peered":"2024-07-28T19:49:14.411877+0000","last_clean":"2024-07-28T19:49:14.411877+0000","last_became_active":"2024-07-28T19:42:51.424962+0000","last_became_peered":"2024-07-28T19:42:51.424962+0000","last_unstale":"2024-07-28T19:49:14.411877+0000","last_undegraded":"2024-07-28T19:49:14.411877+0000","last_fullsized":"2024-07-28T19:49:14.411877+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:12:35.969954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":906,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.736508+0000","last_change":"2024-07-28T19:41:51.007682+0000","last_active":"2024-07-28T19:49:15.736508+0000","last_peered":"2024-07-28T19:49:15.736508+0000","last_clean":"2024-07-28T19:49:15.736508+0000","last_became_active":"2024-07-28T19:41:51.007086+0000","last_became_peered":"2024-07-28T19:41:51.007086+0000","last_unstale":"2024-07-28T19:49:15.736508+0000","last_undegraded":"2024-07-28T19:49:15.736508+0000","last_fullsized":"2024-07-28T19:49:15.736508+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:14:39.168420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"379'23","reported_seq":917,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.300264+0000","last_change":"2024-07-28T19:42:51.421626+0000","last_active":"2024-07-28T19:49:15.300264+0000","last_peered":"2024-07-28T19:49:15.300264+0000","last_clean":"2024-07-28T19:49:15.300264+0000","last_became_active":"2024-07-28T19:42:51.418634+0000","last_became_peered":"2024-07-28T19:42:51.418634+0000","last_unstale":"2024-07-28T19:49:15.300264+0000","last_undegraded":"2024-07-28T19:49:15.300264+0000","last_fullsized":"2024-07-28T19:49:15.300264+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:06:51.752498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":903,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.303038+0000","last_change":"2024-07-28T19:42:51.425291+0000","last_active":"2024-07-28T19:49:15.303038+0000","last_peered":"2024-07-28T19:49:15.303038+0000","last_clean":"2024-07-28T19:49:15.303038+0000","last_became_active":"2024-07-28T19:42:51.421933+0000","last_became_peered":"2024-07-28T19:42:51.421933+0000","last_unstale":"2024-07-28T19:49:15.303038+0000","last_undegraded":"2024-07-28T19:49:15.303038+0000","last_fullsized":"2024-07-28T19:49:15.303038+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:32:25.468432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":909,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.413024+0000","last_change":"2024-07-28T19:42:51.398708+0000","last_active":"2024-07-28T19:49:14.413024+0000","last_peered":"2024-07-28T19:49:14.413024+0000","last_clean":"2024-07-28T19:49:14.413024+0000","last_became_active":"2024-07-28T19:42:51.394596+0000","last_became_peered":"2024-07-28T19:42:51.394596+0000","last_unstale":"2024-07-28T19:49:14.413024+0000","last_undegraded":"2024-07-28T19:49:14.413024+0000","last_fullsized":"2024-07-28T19:49:14.413024+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:30:35.750535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":915,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.419706+0000","last_change":"2024-07-28T19:41:50.981938+0000","last_active":"2024-07-28T19:49:14.419706+0000","last_peered":"2024-07-28T19:49:14.419706+0000","last_clean":"2024-07-28T19:49:14.419706+0000","last_became_active":"2024-07-28T19:41:50.981204+0000","last_became_peered":"2024-07-28T19:41:50.981204+0000","last_unstale":"2024-07-28T19:49:14.419706+0000","last_undegraded":"2024-07-28T19:49:14.419706+0000","last_fullsized":"2024-07-28T19:49:14.419706+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:32:36.925368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":921,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.029204+0000","last_change":"2024-07-28T19:42:51.379059+0000","last_active":"2024-07-28T19:49:16.029204+0000","last_peered":"2024-07-28T19:49:16.029204+0000","last_clean":"2024-07-28T19:49:16.029204+0000","last_became_active":"2024-07-28T19:42:51.378354+0000","last_became_peered":"2024-07-28T19:42:51.378354+0000","last_unstale":"2024-07-28T19:49:16.029204+0000","last_undegraded":"2024-07-28T19:49:16.029204+0000","last_fullsized":"2024-07-28T19:49:16.029204+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:51:45.854172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":915,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.292004+0000","last_change":"2024-07-28T19:42:19.278084+0000","last_active":"2024-07-28T19:49:15.292004+0000","last_peered":"2024-07-28T19:49:15.292004+0000","last_clean":"2024-07-28T19:49:15.292004+0000","last_became_active":"2024-07-28T19:42:19.277738+0000","last_became_peered":"2024-07-28T19:42:19.277738+0000","last_unstale":"2024-07-28T19:49:15.292004+0000","last_undegraded":"2024-07-28T19:49:15.292004+0000","last_fullsized":"2024-07-28T19:49:15.292004+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:21:44.055358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":907,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.424176+0000","last_change":"2024-07-28T19:42:51.382107+0000","last_active":"2024-07-28T19:49:14.424176+0000","last_peered":"2024-07-28T19:49:14.424176+0000","last_clean":"2024-07-28T19:49:14.424176+0000","last_became_active":"2024-07-28T19:42:51.381833+0000","last_became_peered":"2024-07-28T19:42:51.381833+0000","last_unstale":"2024-07-28T19:49:14.424176+0000","last_undegraded":"2024-07-28T19:49:14.424176+0000","last_fullsized":"2024-07-28T19:49:14.424176+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:01:56.634928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":918,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.429434+0000","last_change":"2024-07-28T19:42:19.284578+0000","last_active":"2024-07-28T19:49:14.429434+0000","last_peered":"2024-07-28T19:49:14.429434+0000","last_clean":"2024-07-28T19:49:14.429434+0000","last_became_active":"2024-07-28T19:42:19.279079+0000","last_became_peered":"2024-07-28T19:42:19.279079+0000","last_unstale":"2024-07-28T19:49:14.429434+0000","last_undegraded":"2024-07-28T19:49:14.429434+0000","last_fullsized":"2024-07-28T19:49:14.429434+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:39:16.242525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":887,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.028525+0000","last_change":"2024-07-28T19:39:42.015588+0000","last_active":"2024-07-28T19:49:16.028525+0000","last_peered":"2024-07-28T19:49:16.028525+0000","last_clean":"2024-07-28T19:49:16.028525+0000","last_became_active":"2024-07-28T19:39:42.013634+0000","last_became_peered":"2024-07-28T19:39:42.013634+0000","last_unstale":"2024-07-28T19:49:16.028525+0000","last_undegraded":"2024-07-28T19:49:16.028525+0000","last_fullsized":"2024-07-28T19:49:16.028525+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:35:28.512750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":934,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.413610+0000","last_change":"2024-07-28T19:42:51.411817+0000","last_active":"2024-07-28T19:49:14.413610+0000","last_peered":"2024-07-28T19:49:14.413610+0000","last_clean":"2024-07-28T19:49:14.413610+0000","last_became_active":"2024-07-28T19:42:51.411521+0000","last_became_peered":"2024-07-28T19:42:51.411521+0000","last_unstale":"2024-07-28T19:49:14.413610+0000","last_undegraded":"2024-07-28T19:49:14.413610+0000","last_fullsized":"2024-07-28T19:49:14.413610+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T19:55:04.229692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":907,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.737804+0000","last_change":"2024-07-28T19:41:51.004987+0000","last_active":"2024-07-28T19:49:15.737804+0000","last_peered":"2024-07-28T19:49:15.737804+0000","last_clean":"2024-07-28T19:49:15.737804+0000","last_became_active":"2024-07-28T19:41:51.004670+0000","last_became_peered":"2024-07-28T19:41:51.004670+0000","last_unstale":"2024-07-28T19:49:15.737804+0000","last_undegraded":"2024-07-28T19:49:15.737804+0000","last_fullsized":"2024-07-28T19:49:15.737804+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:44:12.419613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":899,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.688723+0000","last_change":"2024-07-28T19:42:51.421653+0000","last_active":"2024-07-28T19:49:15.688723+0000","last_peered":"2024-07-28T19:49:15.688723+0000","last_clean":"2024-07-28T19:49:15.688723+0000","last_became_active":"2024-07-28T19:42:51.421366+0000","last_became_peered":"2024-07-28T19:42:51.421366+0000","last_unstale":"2024-07-28T19:49:15.688723+0000","last_undegraded":"2024-07-28T19:49:15.688723+0000","last_fullsized":"2024-07-28T19:49:15.688723+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:59:56.862003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":902,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.737710+0000","last_change":"2024-07-28T19:40:16.080446+0000","last_active":"2024-07-28T19:49:15.737710+0000","last_peered":"2024-07-28T19:49:15.737710+0000","last_clean":"2024-07-28T19:49:15.737710+0000","last_became_active":"2024-07-28T19:40:16.079427+0000","last_became_peered":"2024-07-28T19:40:16.079427+0000","last_unstale":"2024-07-28T19:49:15.737710+0000","last_undegraded":"2024-07-28T19:49:15.737710+0000","last_fullsized":"2024-07-28T19:49:15.737710+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:01:44.248862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":928,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.416759+0000","last_change":"2024-07-28T19:42:19.270718+0000","last_active":"2024-07-28T19:49:14.416759+0000","last_peered":"2024-07-28T19:49:14.416759+0000","last_clean":"2024-07-28T19:49:14.416759+0000","last_became_active":"2024-07-28T19:42:19.270277+0000","last_became_peered":"2024-07-28T19:42:19.270277+0000","last_unstale":"2024-07-28T19:49:14.416759+0000","last_undegraded":"2024-07-28T19:49:14.416759+0000","last_fullsized":"2024-07-28T19:49:14.416759+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:52:06.646794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":922,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.737917+0000","last_change":"2024-07-28T19:42:19.258262+0000","last_active":"2024-07-28T19:49:15.737917+0000","last_peered":"2024-07-28T19:49:15.737917+0000","last_clean":"2024-07-28T19:49:15.737917+0000","last_became_active":"2024-07-28T19:42:19.257951+0000","last_became_peered":"2024-07-28T19:42:19.257951+0000","last_unstale":"2024-07-28T19:49:15.737917+0000","last_undegraded":"2024-07-28T19:49:15.737917+0000","last_fullsized":"2024-07-28T19:49:15.737917+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:58:19.686714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":911,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.415731+0000","last_change":"2024-07-28T19:42:51.425299+0000","last_active":"2024-07-28T19:49:14.415731+0000","last_peered":"2024-07-28T19:49:14.415731+0000","last_clean":"2024-07-28T19:49:14.415731+0000","last_became_active":"2024-07-28T19:42:51.421774+0000","last_became_peered":"2024-07-28T19:42:51.421774+0000","last_unstale":"2024-07-28T19:49:14.415731+0000","last_undegraded":"2024-07-28T19:49:14.415731+0000","last_fullsized":"2024-07-28T19:49:14.415731+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:18:26.859879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":915,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411546+0000","last_change":"2024-07-28T19:40:16.078698+0000","last_active":"2024-07-28T19:49:14.411546+0000","last_peered":"2024-07-28T19:49:14.411546+0000","last_clean":"2024-07-28T19:49:14.411546+0000","last_became_active":"2024-07-28T19:40:16.077852+0000","last_became_peered":"2024-07-28T19:40:16.077852+0000","last_unstale":"2024-07-28T19:49:14.411546+0000","last_undegraded":"2024-07-28T19:49:14.411546+0000","last_fullsized":"2024-07-28T19:49:14.411546+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:26:06.111600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":896,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.424508+0000","last_change":"2024-07-28T19:40:16.066113+0000","last_active":"2024-07-28T19:49:14.424508+0000","last_peered":"2024-07-28T19:49:14.424508+0000","last_clean":"2024-07-28T19:49:14.424508+0000","last_became_active":"2024-07-28T19:40:16.065767+0000","last_became_peered":"2024-07-28T19:40:16.065767+0000","last_unstale":"2024-07-28T19:49:14.424508+0000","last_undegraded":"2024-07-28T19:49:14.424508+0000","last_fullsized":"2024-07-28T19:49:14.424508+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:18:45.882295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":907,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.734117+0000","last_change":"2024-07-28T19:41:51.001335+0000","last_active":"2024-07-28T19:49:15.734117+0000","last_peered":"2024-07-28T19:49:15.734117+0000","last_clean":"2024-07-28T19:49:15.734117+0000","last_became_active":"2024-07-28T19:41:51.000000+0000","last_became_peered":"2024-07-28T19:41:51.000000+0000","last_unstale":"2024-07-28T19:49:15.734117+0000","last_undegraded":"2024-07-28T19:49:15.734117+0000","last_fullsized":"2024-07-28T19:49:15.734117+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:08:38.692771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":909,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.688996+0000","last_change":"2024-07-28T19:42:19.238407+0000","last_active":"2024-07-28T19:49:15.688996+0000","last_peered":"2024-07-28T19:49:15.688996+0000","last_clean":"2024-07-28T19:49:15.688996+0000","last_became_active":"2024-07-28T19:42:19.238024+0000","last_became_peered":"2024-07-28T19:42:19.238024+0000","last_unstale":"2024-07-28T19:49:15.688996+0000","last_undegraded":"2024-07-28T19:49:15.688996+0000","last_fullsized":"2024-07-28T19:49:15.688996+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:27:02.041697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":916,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.030435+0000","last_change":"2024-07-28T19:41:50.993084+0000","last_active":"2024-07-28T19:49:16.030435+0000","last_peered":"2024-07-28T19:49:16.030435+0000","last_clean":"2024-07-28T19:49:16.030435+0000","last_became_active":"2024-07-28T19:41:50.992504+0000","last_became_peered":"2024-07-28T19:41:50.992504+0000","last_unstale":"2024-07-28T19:49:16.030435+0000","last_undegraded":"2024-07-28T19:49:16.030435+0000","last_fullsized":"2024-07-28T19:49:16.030435+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:17:34.418222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":916,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.027040+0000","last_change":"2024-07-28T19:42:51.392011+0000","last_active":"2024-07-28T19:49:16.027040+0000","last_peered":"2024-07-28T19:49:16.027040+0000","last_clean":"2024-07-28T19:49:16.027040+0000","last_became_active":"2024-07-28T19:42:51.391619+0000","last_became_peered":"2024-07-28T19:42:51.391619+0000","last_unstale":"2024-07-28T19:49:16.027040+0000","last_undegraded":"2024-07-28T19:49:16.027040+0000","last_fullsized":"2024-07-28T19:49:16.027040+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:52:41.939340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":934,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.414256+0000","last_change":"2024-07-28T19:40:16.069978+0000","last_active":"2024-07-28T19:49:14.414256+0000","last_peered":"2024-07-28T19:49:14.414256+0000","last_clean":"2024-07-28T19:49:14.414256+0000","last_became_active":"2024-07-28T19:40:16.068975+0000","last_became_peered":"2024-07-28T19:40:16.068975+0000","last_unstale":"2024-07-28T19:49:14.414256+0000","last_undegraded":"2024-07-28T19:49:14.414256+0000","last_fullsized":"2024-07-28T19:49:14.414256+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T21:17:58.077529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":910,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.419568+0000","last_change":"2024-07-28T19:42:51.416800+0000","last_active":"2024-07-28T19:49:14.419568+0000","last_peered":"2024-07-28T19:49:14.419568+0000","last_clean":"2024-07-28T19:49:14.419568+0000","last_became_active":"2024-07-28T19:42:51.416450+0000","last_became_peered":"2024-07-28T19:42:51.416450+0000","last_unstale":"2024-07-28T19:49:14.419568+0000","last_undegraded":"2024-07-28T19:49:14.419568+0000","last_fullsized":"2024-07-28T19:49:14.419568+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:58:51.835822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":869,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.299724+0000","last_change":"2024-07-28T19:42:19.269576+0000","last_active":"2024-07-28T19:49:15.299724+0000","last_peered":"2024-07-28T19:49:15.299724+0000","last_clean":"2024-07-28T19:49:15.299724+0000","last_became_active":"2024-07-28T19:42:19.266463+0000","last_became_peered":"2024-07-28T19:42:19.266463+0000","last_unstale":"2024-07-28T19:49:15.299724+0000","last_undegraded":"2024-07-28T19:49:15.299724+0000","last_fullsized":"2024-07-28T19:49:15.299724+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:00:16.617549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":915,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411986+0000","last_change":"2024-07-28T19:41:51.000919+0000","last_active":"2024-07-28T19:49:14.411986+0000","last_peered":"2024-07-28T19:49:14.411986+0000","last_clean":"2024-07-28T19:49:14.411986+0000","last_became_active":"2024-07-28T19:41:51.000578+0000","last_became_peered":"2024-07-28T19:41:51.000578+0000","last_unstale":"2024-07-28T19:49:14.411986+0000","last_undegraded":"2024-07-28T19:49:14.411986+0000","last_fullsized":"2024-07-28T19:49:14.411986+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:27:14.523338+0000","last_clean_scrub_stamp":"2024-07-28T19:27:14.523338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:09:59.009367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"305'161","reported_seq":1293,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.413810+0000","last_change":"2024-07-28T19:39:42.005813+0000","last_active":"2024-07-28T19:49:14.413810+0000","last_peered":"2024-07-28T19:49:14.413810+0000","last_clean":"2024-07-28T19:49:14.413810+0000","last_became_active":"2024-07-28T19:39:42.004739+0000","last_became_peered":"2024-07-28T19:39:42.004739+0000","last_unstale":"2024-07-28T19:49:14.413810+0000","last_undegraded":"2024-07-28T19:49:14.413810+0000","last_fullsized":"2024-07-28T19:49:14.413810+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:21:20.215133+0000","last_clean_scrub_stamp":"2024-07-28T19:21:20.215133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:25:39.147946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":921,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.424813+0000","last_change":"2024-07-28T19:42:51.399069+0000","last_active":"2024-07-28T19:49:14.424813+0000","last_peered":"2024-07-28T19:49:14.424813+0000","last_clean":"2024-07-28T19:49:14.424813+0000","last_became_active":"2024-07-28T19:42:51.398615+0000","last_became_peered":"2024-07-28T19:42:51.398615+0000","last_unstale":"2024-07-28T19:49:14.424813+0000","last_undegraded":"2024-07-28T19:49:14.424813+0000","last_fullsized":"2024-07-28T19:49:14.424813+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:20:40.672584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":909,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.427640+0000","last_change":"2024-07-28T19:42:19.278426+0000","last_active":"2024-07-28T19:49:14.427640+0000","last_peered":"2024-07-28T19:49:14.427640+0000","last_clean":"2024-07-28T19:49:14.427640+0000","last_became_active":"2024-07-28T19:42:19.278133+0000","last_became_peered":"2024-07-28T19:42:19.278133+0000","last_unstale":"2024-07-28T19:49:14.427640+0000","last_undegraded":"2024-07-28T19:49:14.427640+0000","last_fullsized":"2024-07-28T19:49:14.427640+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:28:49.284657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":926,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.304360+0000","last_change":"2024-07-28T19:42:51.404654+0000","last_active":"2024-07-28T19:49:15.304360+0000","last_peered":"2024-07-28T19:49:15.304360+0000","last_clean":"2024-07-28T19:49:15.304360+0000","last_became_active":"2024-07-28T19:42:51.404064+0000","last_became_peered":"2024-07-28T19:42:51.404064+0000","last_unstale":"2024-07-28T19:49:15.304360+0000","last_undegraded":"2024-07-28T19:49:15.304360+0000","last_fullsized":"2024-07-28T19:49:15.304360+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:12:13.730589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":927,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.299575+0000","last_change":"2024-07-28T19:42:51.408176+0000","last_active":"2024-07-28T19:49:15.299575+0000","last_peered":"2024-07-28T19:49:15.299575+0000","last_clean":"2024-07-28T19:49:15.299575+0000","last_became_active":"2024-07-28T19:42:51.403879+0000","last_became_peered":"2024-07-28T19:42:51.403879+0000","last_unstale":"2024-07-28T19:49:15.299575+0000","last_undegraded":"2024-07-28T19:49:15.299575+0000","last_fullsized":"2024-07-28T19:49:15.299575+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:21:19.797025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":925,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.301376+0000","last_change":"2024-07-28T19:42:51.416379+0000","last_active":"2024-07-28T19:49:15.301376+0000","last_peered":"2024-07-28T19:49:15.301376+0000","last_clean":"2024-07-28T19:49:15.301376+0000","last_became_active":"2024-07-28T19:42:51.412668+0000","last_became_peered":"2024-07-28T19:42:51.412668+0000","last_unstale":"2024-07-28T19:49:15.301376+0000","last_undegraded":"2024-07-28T19:49:15.301376+0000","last_fullsized":"2024-07-28T19:49:15.301376+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T00:40:00.497141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":928,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.419000+0000","last_change":"2024-07-28T19:42:51.396048+0000","last_active":"2024-07-28T19:49:14.419000+0000","last_peered":"2024-07-28T19:49:14.419000+0000","last_clean":"2024-07-28T19:49:14.419000+0000","last_became_active":"2024-07-28T19:42:51.395490+0000","last_became_peered":"2024-07-28T19:42:51.395490+0000","last_unstale":"2024-07-28T19:49:14.419000+0000","last_undegraded":"2024-07-28T19:49:14.419000+0000","last_fullsized":"2024-07-28T19:49:14.419000+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:05:38.634689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":923,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.300285+0000","last_change":"2024-07-28T19:42:51.418278+0000","last_active":"2024-07-28T19:49:15.300285+0000","last_peered":"2024-07-28T19:49:15.300285+0000","last_clean":"2024-07-28T19:49:15.300285+0000","last_became_active":"2024-07-28T19:42:51.406218+0000","last_became_peered":"2024-07-28T19:42:51.406218+0000","last_unstale":"2024-07-28T19:49:15.300285+0000","last_undegraded":"2024-07-28T19:49:15.300285+0000","last_fullsized":"2024-07-28T19:49:15.300285+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T01:27:45.256338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":934,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.429207+0000","last_change":"2024-07-28T19:42:51.444620+0000","last_active":"2024-07-28T19:49:14.429207+0000","last_peered":"2024-07-28T19:49:14.429207+0000","last_clean":"2024-07-28T19:49:14.429207+0000","last_became_active":"2024-07-28T19:42:51.444220+0000","last_became_peered":"2024-07-28T19:42:51.444220+0000","last_unstale":"2024-07-28T19:49:14.429207+0000","last_undegraded":"2024-07-28T19:49:14.429207+0000","last_fullsized":"2024-07-28T19:49:14.429207+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:15:06.177140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":916,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.424919+0000","last_change":"2024-07-28T19:42:51.432640+0000","last_active":"2024-07-28T19:49:14.424919+0000","last_peered":"2024-07-28T19:49:14.424919+0000","last_clean":"2024-07-28T19:49:14.424919+0000","last_became_active":"2024-07-28T19:42:51.431525+0000","last_became_peered":"2024-07-28T19:42:51.431525+0000","last_unstale":"2024-07-28T19:49:14.424919+0000","last_undegraded":"2024-07-28T19:49:14.424919+0000","last_fullsized":"2024-07-28T19:49:14.424919+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T04:10:09.617690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":911,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.415777+0000","last_change":"2024-07-28T19:42:19.272179+0000","last_active":"2024-07-28T19:49:14.415777+0000","last_peered":"2024-07-28T19:49:14.415777+0000","last_clean":"2024-07-28T19:49:14.415777+0000","last_became_active":"2024-07-28T19:42:19.271578+0000","last_became_peered":"2024-07-28T19:42:19.271578+0000","last_unstale":"2024-07-28T19:49:14.415777+0000","last_undegraded":"2024-07-28T19:49:14.415777+0000","last_fullsized":"2024-07-28T19:49:14.415777+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:21:34.865305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":910,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.418727+0000","last_change":"2024-07-28T19:40:16.066388+0000","last_active":"2024-07-28T19:49:14.418727+0000","last_peered":"2024-07-28T19:49:14.418727+0000","last_clean":"2024-07-28T19:49:14.418727+0000","last_became_active":"2024-07-28T19:40:16.065619+0000","last_became_peered":"2024-07-28T19:40:16.065619+0000","last_unstale":"2024-07-28T19:49:14.418727+0000","last_undegraded":"2024-07-28T19:49:14.418727+0000","last_fullsized":"2024-07-28T19:49:14.418727+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:15:53.247278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":898,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.413337+0000","last_change":"2024-07-28T19:29:15.279653+0000","last_active":"2024-07-28T19:49:14.413337+0000","last_peered":"2024-07-28T19:49:14.413337+0000","last_clean":"2024-07-28T19:49:14.413337+0000","last_became_active":"2024-07-28T19:29:15.278829+0000","last_became_peered":"2024-07-28T19:29:15.278829+0000","last_unstale":"2024-07-28T19:49:14.413337+0000","last_undegraded":"2024-07-28T19:49:14.413337+0000","last_fullsized":"2024-07-28T19:49:14.413337+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T02:46:07.127995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":943,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.299793+0000","last_change":"2024-07-28T19:42:51.377351+0000","last_active":"2024-07-28T19:49:15.299793+0000","last_peered":"2024-07-28T19:49:15.299793+0000","last_clean":"2024-07-28T19:49:15.299793+0000","last_became_active":"2024-07-28T19:42:51.373120+0000","last_became_peered":"2024-07-28T19:42:51.373120+0000","last_unstale":"2024-07-28T19:49:15.299793+0000","last_undegraded":"2024-07-28T19:49:15.299793+0000","last_fullsized":"2024-07-28T19:49:15.299793+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T07:24:25.710095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":933,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.414772+0000","last_change":"2024-07-28T19:42:51.413620+0000","last_active":"2024-07-28T19:49:14.414772+0000","last_peered":"2024-07-28T19:49:14.414772+0000","last_clean":"2024-07-28T19:49:14.414772+0000","last_became_active":"2024-07-28T19:42:51.412048+0000","last_became_peered":"2024-07-28T19:42:51.412048+0000","last_unstale":"2024-07-28T19:49:14.414772+0000","last_undegraded":"2024-07-28T19:49:14.414772+0000","last_fullsized":"2024-07-28T19:49:14.414772+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:05:18.319837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":893,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411864+0000","last_change":"2024-07-28T19:29:16.445468+0000","last_active":"2024-07-28T19:49:14.411864+0000","last_peered":"2024-07-28T19:49:14.411864+0000","last_clean":"2024-07-28T19:49:14.411864+0000","last_became_active":"2024-07-28T19:29:16.444693+0000","last_became_peered":"2024-07-28T19:29:16.444693+0000","last_unstale":"2024-07-28T19:49:14.411864+0000","last_undegraded":"2024-07-28T19:49:14.411864+0000","last_fullsized":"2024-07-28T19:49:14.411864+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:13:39.927285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":932,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.415260+0000","last_change":"2024-07-28T19:42:51.411510+0000","last_active":"2024-07-28T19:49:14.415260+0000","last_peered":"2024-07-28T19:49:14.415260+0000","last_clean":"2024-07-28T19:49:14.415260+0000","last_became_active":"2024-07-28T19:42:51.411189+0000","last_became_peered":"2024-07-28T19:42:51.411189+0000","last_unstale":"2024-07-28T19:49:14.415260+0000","last_undegraded":"2024-07-28T19:49:14.415260+0000","last_fullsized":"2024-07-28T19:49:14.415260+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:27:19.712350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":923,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.418595+0000","last_change":"2024-07-28T19:39:41.991183+0000","last_active":"2024-07-28T19:49:14.418595+0000","last_peered":"2024-07-28T19:49:14.418595+0000","last_clean":"2024-07-28T19:49:14.418595+0000","last_became_active":"2024-07-28T19:39:41.990138+0000","last_became_peered":"2024-07-28T19:39:41.990138+0000","last_unstale":"2024-07-28T19:49:14.418595+0000","last_undegraded":"2024-07-28T19:49:14.418595+0000","last_fullsized":"2024-07-28T19:49:14.418595+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T05:59:07.444830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":908,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.293779+0000","last_change":"2024-07-28T19:42:19.284241+0000","last_active":"2024-07-28T19:49:15.293779+0000","last_peered":"2024-07-28T19:49:15.293779+0000","last_clean":"2024-07-28T19:49:15.293779+0000","last_became_active":"2024-07-28T19:42:19.281556+0000","last_became_peered":"2024-07-28T19:42:19.281556+0000","last_unstale":"2024-07-28T19:49:15.293779+0000","last_undegraded":"2024-07-28T19:49:15.293779+0000","last_fullsized":"2024-07-28T19:49:15.293779+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T20:02:57.369978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":913,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.411991+0000","last_change":"2024-07-28T19:42:51.401979+0000","last_active":"2024-07-28T19:49:14.411991+0000","last_peered":"2024-07-28T19:49:14.411991+0000","last_clean":"2024-07-28T19:49:14.411991+0000","last_became_active":"2024-07-28T19:42:51.401314+0000","last_became_peered":"2024-07-28T19:42:51.401314+0000","last_unstale":"2024-07-28T19:49:14.411991+0000","last_undegraded":"2024-07-28T19:49:14.411991+0000","last_fullsized":"2024-07-28T19:49:14.411991+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:42:31.056894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":901,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:16.028696+0000","last_change":"2024-07-28T19:42:19.274756+0000","last_active":"2024-07-28T19:49:16.028696+0000","last_peered":"2024-07-28T19:49:16.028696+0000","last_clean":"2024-07-28T19:49:16.028696+0000","last_became_active":"2024-07-28T19:42:19.273800+0000","last_became_peered":"2024-07-28T19:42:19.273800+0000","last_unstale":"2024-07-28T19:49:16.028696+0000","last_undegraded":"2024-07-28T19:49:16.028696+0000","last_fullsized":"2024-07-28T19:49:16.028696+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:33:42.924762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":900,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.426414+0000","last_change":"2024-07-28T19:42:19.233491+0000","last_active":"2024-07-28T19:49:14.426414+0000","last_peered":"2024-07-28T19:49:14.426414+0000","last_clean":"2024-07-28T19:49:14.426414+0000","last_became_active":"2024-07-28T19:42:19.233074+0000","last_became_peered":"2024-07-28T19:42:19.233074+0000","last_unstale":"2024-07-28T19:49:14.426414+0000","last_undegraded":"2024-07-28T19:49:14.426414+0000","last_fullsized":"2024-07-28T19:49:14.426414+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:12:05.408285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":910,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.417079+0000","last_change":"2024-07-28T19:42:19.269055+0000","last_active":"2024-07-28T19:49:14.417079+0000","last_peered":"2024-07-28T19:49:14.417079+0000","last_clean":"2024-07-28T19:49:14.417079+0000","last_became_active":"2024-07-28T19:42:19.268658+0000","last_became_peered":"2024-07-28T19:42:19.268658+0000","last_unstale":"2024-07-28T19:49:14.417079+0000","last_undegraded":"2024-07-28T19:49:14.417079+0000","last_fullsized":"2024-07-28T19:49:14.417079+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-30T03:42:01.885987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":934,"reported_epoch":960,"state":"active+clean","last_fresh":"2024-07-28T19:49:14.412795+0000","last_change":"2024-07-28T19:42:51.429453+0000","last_active":"2024-07-28T19:49:14.412795+0000","last_peered":"2024-07-28T19:49:14.412795+0000","last_clean":"2024-07-28T19:49:14.412795+0000","last_became_active":"2024-07-28T19:42:51.428701+0000","last_became_peered":"2024-07-28T19:42:51.428701+0000","last_unstale":"2024-07-28T19:49:14.412795+0000","last_undegraded":"2024-07-28T19:49:14.412795+0000","last_fullsized":"2024-07-28T19:49:14.412795+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:14.223452+0000","last_clean_scrub_stamp":"2024-07-28T19:29:14.223452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T22:21:30.803400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":923,"reported_epoch":961,"state":"active+clean","last_fresh":"2024-07-28T19:49:15.293558+0000","last_change":"2024-07-28T19:42:19.291360+0000","last_active":"2024-07-28T19:49:15.293558+0000","last_peered":"2024-07-28T19:49:15.293558+0000","last_clean":"2024-07-28T19:49:15.293558+0000","last_became_active":"2024-07-28T19:42:19.289985+0000","last_became_peered":"2024-07-28T19:42:19.289985+0000","last_unstale":"2024-07-28T19:49:15.293558+0000","last_undegraded":"2024-07-28T19:49:15.293558+0000","last_fullsized":"2024-07-28T19:49:15.293558+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-07-28T19:29:15.227012+0000","last_clean_scrub_stamp":"2024-07-28T19:29:15.227012+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-07-29T23:03:28.790331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5542,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5895,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201587,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308928,"kb_used_data":11576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93436224,"statfs":{"total":95995035648,"available":95678693376,"internally_reserved":0,"allocated":11853824,"data_stored":7760665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.1859999999999999,"15min":1.0620000000000001},"min":{"1min":0.42999999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.26,"5min":17.626000000000001,"15min":17.626000000000001},"last":1.155},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.4570000000000001,"15min":1.28},"min":{"1min":0.57299999999999995,"5min":0.375,"15min":0.375},"max":{"1min":1.2,"5min":29.181000000000001,"15min":29.181000000000001},"last":1.0680000000000001}]},{"osd":1,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.98099999999999998,"15min":0.98799999999999999},"min":{"1min":0.46400000000000002,"5min":0.45000000000000001,"15min":0.28299999999999997},"max":{"1min":1.151,"5min":12.973000000000001,"15min":14.095000000000001},"last":1.369},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.2450000000000001,"15min":1.1679999999999999},"min":{"1min":0.53800000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.306,"5min":29.052,"15min":29.052},"last":1.21}]},{"osd":2,"last update":"Sun Jul 28 19:49:59 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.411,"15min":1.2090000000000001},"min":{"1min":0.48799999999999999,"5min":0.41099999999999998,"15min":0.38600000000000001},"max":{"1min":1.3560000000000001,"5min":29.131,"15min":29.131},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.996,"15min":0.91900000000000004},"min":{"1min":0.47599999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.2010000000000001,"5min":7.7300000000000004,"15min":7.7300000000000004},"last":0.90700000000000003}]},{"osd":3,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.0489999999999999,"15min":1.0129999999999999},"min":{"1min":0.441,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.177,"5min":7.5590000000000002,"15min":13.332000000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.86799999999999999,"5min":1.115,"15min":1.4239999999999999},"min":{"1min":0.50700000000000001,"5min":0.495,"15min":0.495},"max":{"1min":1.1559999999999999,"5min":13.119,"15min":14.263},"last":1.288}]},{"osd":4,"last update":"Sun Jul 28 19:49:59 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.4630000000000001,"15min":1.143},"min":{"1min":0.55800000000000005,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.3480000000000001,"5min":29.547000000000001,"15min":29.547000000000001},"last":1.4359999999999999},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.452,"15min":1.2330000000000001},"min":{"1min":0.57999999999999996,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2769999999999999,"5min":29.263000000000002,"15min":29.263000000000002},"last":0.85699999999999998}]},{"osd":5,"last update":"Sun Jul 28 19:50:28 2024","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.99299999999999999,"15min":1.1459999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.34000000000000002},"max":{"1min":1.284,"5min":7.657,"15min":33.914000000000001},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.014,"5min":1.6080000000000001,"15min":1.4039999999999999},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.376},"max":{"1min":1.1779999999999999,"5min":38.695,"15min":38.695},"last":1.569}]},{"osd":6,"last update":"Sun Jul 28 19:50:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.27,"15min":1.345},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.35299999999999998},"max":{"1min":1.3140000000000001,"5min":13.042999999999999,"15min":27.559000000000001},"last":1.5},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.502,"15min":1.46},"min":{"1min":0.52400000000000002,"5min":0.41199999999999998,"15min":0.36799999999999999},"max":{"1min":1.218,"5min":29.218,"15min":31.202999999999999},"last":1.262}]}]},{"osd":6,"up_from":47,"seq":201863463229,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317240,"kb_used_data":11564,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93427912,"statfs":{"total":95995035648,"available":95670181888,"internally_reserved":0,"allocated":11841536,"data_stored":7760519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.94099999999999995,"15min":0.88500000000000001},"min":{"1min":0.55800000000000005,"5min":0.504,"15min":0.42999999999999999},"max":{"1min":1.3140000000000001,"5min":3.7069999999999999,"15min":3.7069999999999999},"last":0.752},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.95599999999999996,"15min":0.997},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.35999999999999999},"max":{"1min":1.149,"5min":4.1950000000000003,"15min":16.055},"last":1.0580000000000001}]},{"osd":1,"last update":"Sun Jul 28 19:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.98199999999999998,"15min":0.96999999999999997},"min":{"1min":0.497,"5min":0.497,"15min":0.377},"max":{"1min":1.204,"5min":4.4130000000000003,"15min":4.4130000000000003},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.996,"5min":1.0349999999999999,"15min":0.97899999999999998},"min":{"1min":0.53800000000000003,"5min":0.47199999999999998,"15min":0.378},"max":{"1min":1.4199999999999999,"5min":3.9079999999999999,"15min":3.9079999999999999},"last":1.1759999999999999}]},{"osd":2,"last update":"Sun Jul 28 19:50:09 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.0109999999999999,"15min":1.075},"min":{"1min":0.54500000000000004,"5min":0.437,"15min":0.437},"max":{"1min":1.2649999999999999,"5min":4.3579999999999997,"15min":4.4050000000000002},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.179,"15min":1.046},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2569999999999999,"5min":6.2380000000000004,"15min":6.2380000000000004},"last":0.91600000000000004}]},{"osd":3,"last update":"Sun Jul 28 19:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.097,"15min":1.0640000000000001},"min":{"1min":0.59299999999999997,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.242,"5min":5.7220000000000004,"15min":5.7220000000000004},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.0960000000000001,"15min":1.1379999999999999},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":1.194,"5min":4.3029999999999999,"15min":4.6870000000000003},"last":1.1559999999999999}]},{"osd":4,"last update":"Sun Jul 28 19:50:09 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.1659999999999999,"15min":1.03},"min":{"1min":0.48199999999999998,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.179,"5min":6.141,"15min":6.141},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.004,"5min":1.274,"15min":1.1950000000000001},"min":{"1min":0.67000000000000004,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.3100000000000001,"5min":5.8780000000000001,"15min":5.8780000000000001},"last":1.3220000000000001}]},{"osd":5,"last update":"Sun Jul 28 19:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":1.117,"15min":1.048},"min":{"1min":0.51900000000000002,"5min":0.39000000000000001,"15min":0.316},"max":{"1min":1.155,"5min":6.5659999999999998,"15min":6.5659999999999998},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.002,"5min":1.139,"15min":1.0860000000000001},"min":{"1min":0.45900000000000002,"5min":0.441,"15min":0.40400000000000003},"max":{"1min":1.226,"5min":5.9720000000000004,"15min":5.9720000000000004},"last":1.0880000000000001}]},{"osd":7,"last update":"Sun Jul 28 19:50:21 2024","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":1.222,"15min":1.1479999999999999},"min":{"1min":0.68400000000000005,"5min":0.42099999999999999,"15min":0.33700000000000002},"max":{"1min":1.2190000000000001,"5min":6.3140000000000001,"15min":7.8620000000000001},"last":1.026},{"interface":"front","average":{"1min":1.052,"5min":1.222,"15min":1.181},"min":{"1min":0.58199999999999996,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.335,"5min":5.6299999999999999,"15min":8.6349999999999998},"last":1.254}]}]},{"osd":5,"up_from":39,"seq":167503724871,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":308332,"kb_used_data":10976,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93436820,"statfs":{"total":95995035648,"available":95679303680,"internally_reserved":0,"allocated":11239424,"data_stored":7165965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.83499999999999996,"15min":0.89000000000000001},"min":{"1min":0.40999999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.9650000000000001,"5min":2.2400000000000002,"15min":13.148},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.96899999999999997,"15min":0.90500000000000003},"min":{"1min":0.50900000000000001,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":4.1859999999999999,"5min":6.1139999999999999,"15min":6.1139999999999999},"last":0.97399999999999998}]},{"osd":1,"last update":"Sun Jul 28 19:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.014,"15min":1.044},"min":{"1min":0.59999999999999998,"5min":0.58299999999999996,"15min":0.314},"max":{"1min":1.208,"5min":2.4100000000000001,"15min":14.454000000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.05,"5min":1.139,"15min":1.161},"min":{"1min":0.77800000000000002,"5min":0.54400000000000004,"15min":0.40100000000000002},"max":{"1min":1.3460000000000001,"5min":2.528,"15min":14.536},"last":1.083}]},{"osd":2,"last update":"Sun Jul 28 19:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.92300000000000004,"15min":1.22},"min":{"1min":0.61799999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":3.75,"5min":3.75,"15min":12.542},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0449999999999999,"15min":1.3620000000000001},"min":{"1min":0.58799999999999997,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":3.8239999999999998,"5min":3.8239999999999998,"15min":12.727},"last":1.026}]},{"osd":3,"last update":"Sun Jul 28 19:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.048,"15min":0.95799999999999996},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.41499999999999998},"max":{"1min":1.224,"5min":6.1980000000000004,"15min":6.1980000000000004},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.147,"15min":1.0780000000000001},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.47599999999999998},"max":{"1min":1.216,"5min":2.6469999999999998,"15min":2.6469999999999998},"last":1.238}]},{"osd":4,"last update":"Sun Jul 28 19:50:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.073,"15min":1.101},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":2.3410000000000002,"5min":5.5250000000000004,"15min":5.5250000000000004},"last":1.165},{"interface":"front","average":{"1min":1.161,"5min":1.1859999999999999,"15min":1.079},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":3.915,"5min":6.9660000000000002,"15min":6.9660000000000002},"last":1.268}]},{"osd":6,"last update":"Sun Jul 28 19:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.0189999999999999,"15min":1.0529999999999999},"min":{"1min":0.42599999999999999,"5min":0.39200000000000002,"15min":0.33700000000000002},"max":{"1min":1.1850000000000001,"5min":5.2489999999999997,"15min":12.981999999999999},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.087,"5min":1.24,"15min":1.113},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.42699999999999999},"max":{"1min":1.381,"5min":7.0449999999999999,"15min":7.0449999999999999},"last":1.337}]},{"osd":7,"last update":"Sun Jul 28 19:50:21 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.173,"15min":1.105},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.36899999999999999},"max":{"1min":1.419,"5min":7.1070000000000002,"15min":9.6110000000000007},"last":1.115},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.216,"15min":1.1859999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.456,"5min":5.6230000000000002,"15min":15.936},"last":1.306}]}]},{"osd":4,"up_from":652,"seq":2800318677089,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":64952,"kb_used_data":10992,"kb_used_omap":67,"kb_used_meta":53884,"kb_avail":93680200,"statfs":{"total":95995035648,"available":95928524800,"internally_reserved":0,"allocated":11255808,"data_stored":7170227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69313,"internal_metadata":55177535},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.3169999999999999,"15min":0.98499999999999999},"min":{"1min":0.65900000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.827,"5min":20.173999999999999,"15min":20.173999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.071,"15min":0.93300000000000005},"min":{"1min":0.64300000000000002,"5min":0.501,"15min":0.42599999999999999},"max":{"1min":1.8879999999999999,"5min":4.1109999999999998,"15min":4.1109999999999998},"last":1.171}]},{"osd":1,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.996,"15min":0.97199999999999998},"min":{"1min":0.628,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.2010000000000001,"5min":2.3519999999999999,"15min":2.3519999999999999},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.96999999999999997,"15min":1.0249999999999999},"min":{"1min":0.51700000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.274,"5min":1.4399999999999999,"15min":2.1259999999999999},"last":1.288}]},{"osd":2,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.0369999999999999,"15min":1.042},"min":{"1min":0.68600000000000005,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.3180000000000001,"5min":2.5019999999999998,"15min":2.5019999999999998},"last":1.0589999999999999},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.1639999999999999,"15min":1.056},"min":{"1min":0.66800000000000004,"5min":0.504,"15min":0.504},"max":{"1min":1.2829999999999999,"5min":17.503,"15min":17.503},"last":1.038}]},{"osd":3,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.042,"15min":1.004},"min":{"1min":0.75900000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.242,"5min":2.2200000000000002,"15min":2.2200000000000002},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.077,"5min":1.101,"15min":1.1639999999999999},"min":{"1min":0.93000000000000005,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.2230000000000001,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":1.1399999999999999}]},{"osd":5,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0329999999999999,"15min":1.0640000000000001},"min":{"1min":0.77300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2490000000000001,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.1870000000000001,"15min":1.1379999999999999},"min":{"1min":0.60599999999999998,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.373,"5min":17.393999999999998,"15min":17.393999999999998},"last":0.747}]},{"osd":6,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.236,"15min":1.1639999999999999},"min":{"1min":0.66600000000000004,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3979999999999999,"5min":17.664000000000001,"15min":17.664000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.1879999999999999,"15min":1.141},"min":{"1min":0.76500000000000001,"5min":0.55300000000000005,"15min":0.47899999999999998},"max":{"1min":1.363,"5min":7.2220000000000004,"15min":7.2220000000000004},"last":0.79900000000000004}]},{"osd":7,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.1619999999999999,"15min":1.1799999999999999},"min":{"1min":0.73299999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3480000000000001,"5min":4.0170000000000003,"15min":4.0170000000000003},"last":1.2529999999999999},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.2210000000000001,"15min":1.3320000000000001},"min":{"1min":0.89600000000000002,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.4330000000000001,"5min":4.7640000000000002,"15min":4.7640000000000002},"last":1.2070000000000001}]}]},{"osd":3,"up_from":643,"seq":2761663971432,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":92856,"kb_used_data":10972,"kb_used_omap":69,"kb_used_meta":57914,"kb_avail":93652296,"statfs":{"total":95995035648,"available":95899951104,"internally_reserved":0,"allocated":11235328,"data_stored":7165415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70822,"internal_metadata":59304794},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.745,"15min":0.65900000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":0.98399999999999999,"5min":4.0519999999999996,"15min":4.0519999999999996},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.746,"5min":0.88100000000000001,"15min":0.73599999999999999},"min":{"1min":0.44,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":1.0860000000000001,"5min":4.7009999999999996,"15min":6.4370000000000003},"last":0.64300000000000002}]},{"osd":1,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.85099999999999998,"15min":0.80300000000000005},"min":{"1min":0.41599999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.242,"5min":3.177,"15min":3.294},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.88800000000000001,"15min":0.80700000000000005},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.26400000000000001},"max":{"1min":1.038,"5min":8.7010000000000005,"15min":20.555},"last":0.86199999999999999}]},{"osd":2,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.89700000000000002,"15min":0.84099999999999997},"min":{"1min":0.36099999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.169,"5min":5.5250000000000004,"15min":5.5250000000000004},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.90300000000000002,"15min":0.88500000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.29399999999999998},"max":{"1min":1.091,"5min":3.9220000000000002,"15min":20.242000000000001},"last":0.84299999999999997}]},{"osd":4,"last update":"Sun Jul 28 19:50:08 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0429999999999999,"15min":0.98799999999999999},"min":{"1min":0.80900000000000005,"5min":0.501,"15min":0.437},"max":{"1min":1.4239999999999999,"5min":3.5179999999999998,"15min":3.831},"last":0.91000000000000003},{"interface":"front","average":{"1min":1.026,"5min":1.046,"15min":1.0640000000000001},"min":{"1min":0.72299999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.4119999999999999,"5min":3.3159999999999998,"15min":20.440000000000001},"last":0.94899999999999995}]},{"osd":5,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.0620000000000001,"15min":1.04},"min":{"1min":0.755,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.2310000000000001,"5min":8.8109999999999999,"15min":23.391999999999999},"last":1.4850000000000001},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.1040000000000001,"15min":1.036},"min":{"1min":0.65900000000000003,"5min":0.51700000000000002,"15min":0.39400000000000002},"max":{"1min":1.415,"5min":3.4529999999999998,"15min":3.4529999999999998},"last":1.0600000000000001}]},{"osd":6,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.2929999999999999,"15min":1.1379999999999999},"min":{"1min":0.629,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":4.1040000000000001,"5min":11.532999999999999,"15min":11.532999999999999},"last":1.143},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1040000000000001,"15min":1.131},"min":{"1min":0.86599999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.5920000000000001,"5min":2.3079999999999998,"15min":20.358000000000001},"last":1.1160000000000001}]},{"osd":7,"last update":"Sun Jul 28 19:50:29 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.169,"15min":1.137},"min":{"1min":0.70599999999999996,"5min":0.442,"15min":0.442},"max":{"1min":1.341,"5min":9.0700000000000003,"15min":21.030000000000001},"last":1.087},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.208,"15min":1.153},"min":{"1min":0.68300000000000005,"5min":0.45900000000000002,"15min":0.41399999999999998},"max":{"1min":4.1929999999999996,"5min":4.1929999999999996,"15min":4.1929999999999996},"last":1.284}]}]},{"osd":2,"up_from":628,"seq":2697239461997,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":97172,"kb_used_data":10976,"kb_used_omap":72,"kb_used_meta":58103,"kb_avail":93647980,"statfs":{"total":95995035648,"available":95895531520,"internally_reserved":0,"allocated":11239424,"data_stored":7165929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74488,"internal_metadata":59497736},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.68000000000000005,"15min":0.59399999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.21099999999999999},"max":{"1min":0.98999999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.749,"5min":0.83899999999999997,"15min":0.70299999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":1.143,"5min":4.5389999999999997,"15min":4.5389999999999997},"last":0.58999999999999997}]},{"osd":1,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":1.018,"15min":0.76400000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.30599999999999999},"max":{"1min":1.038,"5min":14.276999999999999,"15min":14.276999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.73099999999999998,"5min":1.1299999999999999,"15min":0.76700000000000002},"min":{"1min":0.36799999999999999,"5min":0.29499999999999998,"15min":0.24099999999999999},"max":{"1min":1.044,"5min":25.533000000000001,"15min":25.533000000000001},"last":0.76100000000000001}]},{"osd":3,"last update":"Sun Jul 28 19:50:30 2024","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.88300000000000001,"15min":0.80700000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.35099999999999998},"max":{"1min":1.087,"5min":2.754,"15min":2.754},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.877,"15min":0.80500000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.27700000000000002},"max":{"1min":1.0780000000000001,"5min":3.0179999999999998,"15min":3.0179999999999998},"last":0.90400000000000003}]},{"osd":4,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.1180000000000001,"15min":1.2150000000000001},"min":{"1min":0.76300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.2629999999999999,"5min":4.2699999999999996,"15min":5.2990000000000004},"last":0.96399999999999997},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.97299999999999998,"15min":1.1319999999999999},"min":{"1min":0.69099999999999995,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1200000000000001,"5min":1.6100000000000001,"15min":5.3390000000000004},"last":0.82599999999999996}]},{"osd":5,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.0740000000000001,"15min":1.0980000000000001},"min":{"1min":0.64100000000000001,"5min":0.42199999999999999,"15min":0.32600000000000001},"max":{"1min":1.276,"5min":2.4399999999999999,"15min":8.2460000000000004},"last":1.069},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.083,"15min":1.1100000000000001},"min":{"1min":0.82899999999999996,"5min":0.68000000000000005,"15min":0.40100000000000002},"max":{"1min":1.4370000000000001,"5min":2.2120000000000002,"15min":5.1440000000000001},"last":1.0920000000000001}]},{"osd":6,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.1180000000000001,"15min":1.0900000000000001},"min":{"1min":0.73399999999999999,"5min":0.48199999999999998,"15min":0.41399999999999998},"max":{"1min":1.3740000000000001,"5min":2.907,"15min":5.7599999999999998},"last":1.121},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0680000000000001,"15min":1.1499999999999999},"min":{"1min":0.61699999999999999,"5min":0.48999999999999999,"15min":0.36899999999999999},"max":{"1min":1.3380000000000001,"5min":2.657,"15min":8.2119999999999997},"last":1.1419999999999999}]},{"osd":7,"last update":"Sun Jul 28 19:50:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0900000000000001,"15min":1.1879999999999999},"min":{"1min":0.68799999999999994,"5min":0.44,"15min":0.44},"max":{"1min":1.278,"5min":1.7070000000000001,"15min":5.3860000000000001},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.1399999999999999,"15min":1.155},"min":{"1min":0.71599999999999997,"5min":0.71199999999999997,"15min":0.46800000000000003},"max":{"1min":1.3740000000000001,"5min":2.1160000000000001,"15min":4.0579999999999998},"last":1.1839999999999999}]}]},{"osd":1,"up_from":540,"seq":2319282339968,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":60712,"kb_used_data":10968,"kb_used_omap":48,"kb_used_meta":49679,"kb_avail":93684440,"statfs":{"total":95995035648,"available":95932866560,"internally_reserved":0,"allocated":11231232,"data_stored":7165325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49798,"internal_metadata":50871674},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.73399999999999999,"15min":0.72199999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.247},"max":{"1min":0.90700000000000003,"5min":2.7269999999999999,"15min":2.7269999999999999},"last":0.627},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.75600000000000001,"15min":0.71299999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":1.2030000000000001,"5min":4.1909999999999998,"15min":4.1909999999999998},"last":0.67300000000000004}]},{"osd":2,"last update":"Sun Jul 28 19:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.214,"15min":0.88600000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.36499999999999999},"max":{"1min":1.218,"5min":15.945,"15min":15.945},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.107,"15min":0.89900000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.0580000000000001,"5min":15.833,"15min":15.833},"last":0.58499999999999996}]},{"osd":3,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.91800000000000004,"15min":0.84399999999999997},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.246},"max":{"1min":1.0640000000000001,"5min":9.4169999999999998,"15min":9.4169999999999998},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.78600000000000003,"15min":0.88400000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.96799999999999997,"5min":2.0289999999999999,"15min":11.183},"last":0.73099999999999998}]},{"osd":4,"last update":"Sun Jul 28 19:50:07 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.1140000000000001,"15min":1.01},"min":{"1min":0.59999999999999998,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.2829999999999999,"5min":10.903,"15min":10.903},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.1299999999999999,"15min":1.1399999999999999},"min":{"1min":0.65400000000000003,"5min":0.379,"15min":0.379},"max":{"1min":1.238,"5min":11.984999999999999,"15min":18.853000000000002},"last":1.026}]},{"osd":5,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.087,"15min":1.0429999999999999},"min":{"1min":0.56799999999999995,"5min":0.54700000000000004,"15min":0.42899999999999999},"max":{"1min":1.3480000000000001,"5min":4.4640000000000004,"15min":8.2059999999999995},"last":1.054},{"interface":"front","average":{"1min":0.92200000000000004,"5min":1.1240000000000001,"15min":1.044},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.42099999999999999},"max":{"1min":1.1799999999999999,"5min":9.3360000000000003,"15min":9.3360000000000003},"last":0.79000000000000004}]},{"osd":6,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.069,"15min":1.0900000000000001},"min":{"1min":0.60999999999999999,"5min":0.497,"15min":0.497},"max":{"1min":1.268,"5min":3.8250000000000002,"15min":3.8250000000000002},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.1379999999999999,"15min":1.095},"min":{"1min":0.61699999999999999,"5min":0.56499999999999995,"15min":0.50700000000000001},"max":{"1min":1.425,"5min":4.3689999999999998,"15min":4.3689999999999998},"last":1.1120000000000001}]},{"osd":7,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.1060000000000001,"15min":1.129},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.38800000000000001},"max":{"1min":1.2490000000000001,"5min":1.671,"15min":9.0679999999999996},"last":1.081},{"interface":"front","average":{"1min":1.018,"5min":1.1160000000000001,"15min":1.194},"min":{"1min":0.66000000000000003,"5min":0.58099999999999996,"15min":0.52300000000000002},"max":{"1min":1.294,"5min":2.1019999999999999,"15min":11},"last":0.99199999999999999}]}]},{"osd":0,"up_from":506,"seq":2173253451911,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65452,"kb_used_data":11556,"kb_used_omap":55,"kb_used_meta":53832,"kb_avail":93679700,"statfs":{"total":95995035648,"available":95928012800,"internally_reserved":0,"allocated":11833344,"data_stored":7756285,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56449,"internal_metadata":55124863},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.64500000000000002,"15min":0.80200000000000005},"min":{"1min":0.32700000000000001,"5min":0.26500000000000001,"15min":0.22600000000000001},"max":{"1min":1.173,"5min":1.173,"15min":9.5730000000000004},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.746,"15min":0.73799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.272,"5min":1.78,"15min":1.843},"last":0.89000000000000001}]},{"osd":2,"last update":"Sun Jul 28 19:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.76200000000000001,"15min":0.68500000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.24199999999999999},"max":{"1min":1.0289999999999999,"5min":1.8779999999999999,"15min":1.8779999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.83999999999999997,"15min":0.69399999999999995},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29399999999999998},"max":{"1min":2.3090000000000002,"5min":8.3330000000000002,"15min":8.3330000000000002},"last":0.79000000000000004}]},{"osd":3,"last update":"Sun Jul 28 19:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.89400000000000002,"15min":0.88100000000000001},"min":{"1min":0.40799999999999997,"5min":0.35299999999999998,"15min":0.28899999999999998},"max":{"1min":1.131,"5min":4.992,"15min":14.456},"last":0.626},{"interface":"front","average":{"1min":0.82799999999999996,"5min":1.2509999999999999,"15min":1.0229999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.32900000000000001},"max":{"1min":1.3520000000000001,"5min":27.617999999999999,"15min":27.617999999999999},"last":0.92900000000000005}]},{"osd":4,"last update":"Sun Jul 28 19:50:00 2024","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.4510000000000001,"15min":1.242},"min":{"1min":0.68899999999999995,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":2.3519999999999999,"5min":27.690999999999999,"15min":27.690999999999999},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1,"15min":0.96899999999999997},"min":{"1min":0.66900000000000004,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3680000000000001,"5min":2.2919999999999998,"15min":2.2919999999999998},"last":0.998}]},{"osd":5,"last update":"Sun Jul 28 19:50:00 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.96599999999999997,"15min":0.98099999999999998},"min":{"1min":0.50600000000000001,"5min":0.44800000000000001,"15min":0.35599999999999998},"max":{"1min":1.71,"5min":4.9619999999999997,"15min":8.8179999999999996},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.0589999999999999,"15min":1.0329999999999999},"min":{"1min":0.64300000000000002,"5min":0.57999999999999996,"15min":0.38800000000000001},"max":{"1min":1.4490000000000001,"5min":2.3980000000000001,"15min":2.3980000000000001},"last":1.093}]},{"osd":6,"last update":"Sun Jul 28 19:50:00 2024","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.071,"15min":1.0600000000000001},"min":{"1min":0.621,"5min":0.56000000000000005,"15min":0.41499999999999998},"max":{"1min":1.5920000000000001,"5min":2.4750000000000001,"15min":2.4750000000000001},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.6419999999999999,"15min":1.2749999999999999},"min":{"1min":0.68799999999999994,"5min":0.434,"15min":0.42499999999999999},"max":{"1min":2.266,"5min":27.832000000000001,"15min":27.832000000000001},"last":1.127}]},{"osd":7,"last update":"Sun Jul 28 19:50:00 2024","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.488,"15min":1.248},"min":{"1min":0.55400000000000005,"5min":0.49399999999999999,"15min":0.47099999999999997},"max":{"1min":2.4020000000000001,"5min":27.888999999999999,"15min":27.888999999999999},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.05,"15min":1.0820000000000001},"min":{"1min":0.58099999999999996,"5min":0.56999999999999995,"15min":0.44},"max":{"1min":1.784,"5min":2.2080000000000002,"15min":9.3759999999999994},"last":1.018}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1355,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4540,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-07-28T19:50:50.263 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-07-28T19:50:50.263 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-07-28T19:50:50.263 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-07-28T19:50:50.264 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:50:51.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:51 smithi138 bash[67894]: audit 2024-07-28T19:50:49.527376+0000 mgr.y (mgr.25358) 548 : audit [DBG] from='client.34746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:50:51.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:51 smithi138 bash[67894]: cluster 2024-07-28T19:50:50.113424+0000 mgr.y (mgr.25358) 549 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:51.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:51 smithi138 bash[67894]: audit 2024-07-28T19:50:50.471316+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:50:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:51 smithi100 bash[80199]: audit 2024-07-28T19:50:49.527376+0000 mgr.y (mgr.25358) 548 : audit [DBG] from='client.34746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:50:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:51 smithi100 bash[80199]: cluster 2024-07-28T19:50:50.113424+0000 mgr.y (mgr.25358) 549 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:51 smithi100 bash[80199]: audit 2024-07-28T19:50:50.471316+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:50:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:51 smithi100 bash[82644]: audit 2024-07-28T19:50:49.527376+0000 mgr.y (mgr.25358) 548 : audit [DBG] from='client.34746 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-07-28T19:50:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:51 smithi100 bash[82644]: cluster 2024-07-28T19:50:50.113424+0000 mgr.y (mgr.25358) 549 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:51.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:51 smithi100 bash[82644]: audit 2024-07-28T19:50:50.471316+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:50:52.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:50:52 smithi138 bash[60615]: ts=2024-07-28T19:50:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:50:53.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:50:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:50:52] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-07-28T19:50:53.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:53 smithi138 bash[67894]: cluster 2024-07-28T19:50:52.114259+0000 mgr.y (mgr.25358) 550 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:53.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:53 smithi138 bash[67894]: audit 2024-07-28T19:50:52.647873+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:53 smithi100 bash[80199]: cluster 2024-07-28T19:50:52.114259+0000 mgr.y (mgr.25358) 550 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:53 smithi100 bash[80199]: audit 2024-07-28T19:50:52.647873+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:53 smithi100 bash[82644]: cluster 2024-07-28T19:50:52.114259+0000 mgr.y (mgr.25358) 550 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:53.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:53 smithi100 bash[82644]: audit 2024-07-28T19:50:52.647873+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:50:55.043 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:50:55.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:55 smithi100 bash[80199]: cluster 2024-07-28T19:50:54.115280+0000 mgr.y (mgr.25358) 551 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:55.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:55 smithi100 bash[82644]: cluster 2024-07-28T19:50:54.115280+0000 mgr.y (mgr.25358) 551 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:55.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:55 smithi138 bash[67894]: cluster 2024-07-28T19:50:54.115280+0000 mgr.y (mgr.25358) 551 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:56.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[80199]: audit 2024-07-28T19:50:55.733369+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[80199]: audit 2024-07-28T19:50:55.738886+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[80199]: cluster 2024-07-28T19:50:56.115832+0000 mgr.y (mgr.25358) 552 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[80199]: audit 2024-07-28T19:50:56.244879+0000 mon.a (mon.0) 1561 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[80199]: audit 2024-07-28T19:50:56.254781+0000 mon.a (mon.0) 1562 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[80199]: audit 2024-07-28T19:50:56.695557+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[80199]: audit 2024-07-28T19:50:56.697379+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[80199]: audit 2024-07-28T19:50:56.709009+0000 mon.a (mon.0) 1565 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[82644]: audit 2024-07-28T19:50:55.733369+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[82644]: audit 2024-07-28T19:50:55.738886+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:56.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[82644]: cluster 2024-07-28T19:50:56.115832+0000 mgr.y (mgr.25358) 552 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:57.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[82644]: audit 2024-07-28T19:50:56.244879+0000 mon.a (mon.0) 1561 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:57.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[82644]: audit 2024-07-28T19:50:56.254781+0000 mon.a (mon.0) 1562 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:57.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[82644]: audit 2024-07-28T19:50:56.695557+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:50:57.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[82644]: audit 2024-07-28T19:50:56.697379+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:50:57.000 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:56 smithi100 bash[82644]: audit 2024-07-28T19:50:56.709009+0000 mon.a (mon.0) 1565 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:57.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:56 smithi138 bash[67894]: audit 2024-07-28T19:50:55.733369+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:57.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:56 smithi138 bash[67894]: audit 2024-07-28T19:50:55.738886+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:57.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:56 smithi138 bash[67894]: cluster 2024-07-28T19:50:56.115832+0000 mgr.y (mgr.25358) 552 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:57.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:56 smithi138 bash[67894]: audit 2024-07-28T19:50:56.244879+0000 mon.a (mon.0) 1561 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:57.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:56 smithi138 bash[67894]: audit 2024-07-28T19:50:56.254781+0000 mon.a (mon.0) 1562 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:57.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:56 smithi138 bash[67894]: audit 2024-07-28T19:50:56.695557+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:50:57.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:56 smithi138 bash[67894]: audit 2024-07-28T19:50:56.697379+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:50:57.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:56 smithi138 bash[67894]: audit 2024-07-28T19:50:56.709009+0000 mon.a (mon.0) 1565 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:50:58.661 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:50:58.661 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:50:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:59 smithi100 bash[80199]: cluster 2024-07-28T19:50:58.116631+0000 mgr.y (mgr.25358) 553 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:50:59 smithi100 bash[80199]: audit 2024-07-28T19:50:58.662977+0000 mon.a (mon.0) 1566 : audit [DBG] from='client.? 172.21.15.100:0/2296136477' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:50:59.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:59 smithi100 bash[82644]: cluster 2024-07-28T19:50:58.116631+0000 mgr.y (mgr.25358) 553 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:59.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:50:59 smithi100 bash[82644]: audit 2024-07-28T19:50:58.662977+0000 mon.a (mon.0) 1566 : audit [DBG] from='client.? 172.21.15.100:0/2296136477' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:50:59.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:59 smithi138 bash[67894]: cluster 2024-07-28T19:50:58.116631+0000 mgr.y (mgr.25358) 553 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:50:59.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:50:59 smithi138 bash[67894]: audit 2024-07-28T19:50:58.662977+0000 mon.a (mon.0) 1566 : audit [DBG] from='client.? 172.21.15.100:0/2296136477' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:01.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:01 smithi100 bash[80199]: cluster 2024-07-28T19:51:00.117752+0000 mgr.y (mgr.25358) 554 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:01.499 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:01 smithi100 bash[82644]: cluster 2024-07-28T19:51:00.117752+0000 mgr.y (mgr.25358) 554 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:01.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:01 smithi138 bash[67894]: cluster 2024-07-28T19:51:00.117752+0000 mgr.y (mgr.25358) 554 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:02.347 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:51:02.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:51:02 smithi138 bash[60615]: ts=2024-07-28T19:51:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:51:03.181 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:51:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:51:02] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-07-28T19:51:03.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:03 smithi100 bash[80199]: cluster 2024-07-28T19:51:02.118549+0000 mgr.y (mgr.25358) 555 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:03.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:03 smithi100 bash[82644]: cluster 2024-07-28T19:51:02.118549+0000 mgr.y (mgr.25358) 555 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:03 smithi138 bash[67894]: cluster 2024-07-28T19:51:02.118549+0000 mgr.y (mgr.25358) 555 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:05.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:05 smithi100 bash[80199]: cluster 2024-07-28T19:51:04.119440+0000 mgr.y (mgr.25358) 556 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:05.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:05 smithi100 bash[82644]: cluster 2024-07-28T19:51:04.119440+0000 mgr.y (mgr.25358) 556 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:05.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:05 smithi138 bash[67894]: cluster 2024-07-28T19:51:04.119440+0000 mgr.y (mgr.25358) 556 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:07.130 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:51:07.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:07 smithi100 bash[80199]: cluster 2024-07-28T19:51:06.120455+0000 mgr.y (mgr.25358) 557 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:07.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:07 smithi100 bash[82644]: cluster 2024-07-28T19:51:06.120455+0000 mgr.y (mgr.25358) 557 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:07.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:07 smithi138 bash[67894]: cluster 2024-07-28T19:51:06.120455+0000 mgr.y (mgr.25358) 557 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:08.453 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:08 smithi100 bash[80199]: audit 2024-07-28T19:51:07.648394+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:08.453 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:08 smithi100 bash[82644]: audit 2024-07-28T19:51:07.648394+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:08 smithi138 bash[67894]: audit 2024-07-28T19:51:07.648394+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:09.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:09 smithi100 bash[80199]: cluster 2024-07-28T19:51:08.121256+0000 mgr.y (mgr.25358) 558 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:09.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:09 smithi100 bash[82644]: cluster 2024-07-28T19:51:08.121256+0000 mgr.y (mgr.25358) 558 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:09 smithi138 bash[67894]: cluster 2024-07-28T19:51:08.121256+0000 mgr.y (mgr.25358) 558 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:10.829 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:51:10.829 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:51:11.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:11 smithi100 bash[82644]: cluster 2024-07-28T19:51:10.122267+0000 mgr.y (mgr.25358) 559 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:11.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:11 smithi100 bash[82644]: audit 2024-07-28T19:51:10.831027+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.100:0/751729720' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:11.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:11 smithi100 bash[80199]: cluster 2024-07-28T19:51:10.122267+0000 mgr.y (mgr.25358) 559 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:11.499 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:11 smithi100 bash[80199]: audit 2024-07-28T19:51:10.831027+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.100:0/751729720' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:11.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:11 smithi138 bash[67894]: cluster 2024-07-28T19:51:10.122267+0000 mgr.y (mgr.25358) 559 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:11.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:11 smithi138 bash[67894]: audit 2024-07-28T19:51:10.831027+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.100:0/751729720' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:12.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:51:12 smithi138 bash[60615]: ts=2024-07-28T19:51:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:51:13.207 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:51:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:51:12] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-07-28T19:51:13.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:13 smithi100 bash[82644]: cluster 2024-07-28T19:51:12.123009+0000 mgr.y (mgr.25358) 560 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:13.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:13 smithi100 bash[80199]: cluster 2024-07-28T19:51:12.123009+0000 mgr.y (mgr.25358) 560 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:13.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:13 smithi138 bash[67894]: cluster 2024-07-28T19:51:12.123009+0000 mgr.y (mgr.25358) 560 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:14.538 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:51:15.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:15 smithi100 bash[80199]: cluster 2024-07-28T19:51:14.123889+0000 mgr.y (mgr.25358) 561 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:15.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:15 smithi100 bash[82644]: cluster 2024-07-28T19:51:14.123889+0000 mgr.y (mgr.25358) 561 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:15.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:15 smithi138 bash[67894]: cluster 2024-07-28T19:51:14.123889+0000 mgr.y (mgr.25358) 561 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:17.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:17 smithi100 bash[80199]: cluster 2024-07-28T19:51:16.124784+0000 mgr.y (mgr.25358) 562 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:17.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:17 smithi100 bash[82644]: cluster 2024-07-28T19:51:16.124784+0000 mgr.y (mgr.25358) 562 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:17.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:17 smithi138 bash[67894]: cluster 2024-07-28T19:51:16.124784+0000 mgr.y (mgr.25358) 562 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:19.331 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:51:19.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:19 smithi100 bash[80199]: cluster 2024-07-28T19:51:18.125539+0000 mgr.y (mgr.25358) 563 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:19.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:19 smithi100 bash[82644]: cluster 2024-07-28T19:51:18.125539+0000 mgr.y (mgr.25358) 563 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:19.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:19 smithi138 bash[67894]: cluster 2024-07-28T19:51:18.125539+0000 mgr.y (mgr.25358) 563 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:21.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:21 smithi100 bash[80199]: cluster 2024-07-28T19:51:20.126531+0000 mgr.y (mgr.25358) 564 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:21.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:21 smithi100 bash[82644]: cluster 2024-07-28T19:51:20.126531+0000 mgr.y (mgr.25358) 564 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:21 smithi138 bash[67894]: cluster 2024-07-28T19:51:20.126531+0000 mgr.y (mgr.25358) 564 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:22.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:51:22 smithi138 bash[60615]: ts=2024-07-28T19:51:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:51:22.846 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:51:22.846 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:51:23.249 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:51:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:51:22] "GET /metrics HTTP/1.1" 200 34267 "" "Prometheus/2.43.0" 2024-07-28T19:51:23.588 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:23 smithi100 bash[80199]: cluster 2024-07-28T19:51:22.127289+0000 mgr.y (mgr.25358) 565 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:23.588 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:23 smithi100 bash[80199]: audit 2024-07-28T19:51:22.647815+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:23.588 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:23 smithi100 bash[80199]: audit 2024-07-28T19:51:22.848086+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.100:0/3071498274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:23.588 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:23 smithi100 bash[82644]: cluster 2024-07-28T19:51:22.127289+0000 mgr.y (mgr.25358) 565 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:23.588 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:23 smithi100 bash[82644]: audit 2024-07-28T19:51:22.647815+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:23.589 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:23 smithi100 bash[82644]: audit 2024-07-28T19:51:22.848086+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.100:0/3071498274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:23 smithi138 bash[67894]: cluster 2024-07-28T19:51:22.127289+0000 mgr.y (mgr.25358) 565 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:23 smithi138 bash[67894]: audit 2024-07-28T19:51:22.647815+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:23 smithi138 bash[67894]: audit 2024-07-28T19:51:22.848086+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.100:0/3071498274' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:25 smithi100 bash[80199]: cluster 2024-07-28T19:51:24.128189+0000 mgr.y (mgr.25358) 566 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:25 smithi100 bash[82644]: cluster 2024-07-28T19:51:24.128189+0000 mgr.y (mgr.25358) 566 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:25 smithi138 bash[67894]: cluster 2024-07-28T19:51:24.128189+0000 mgr.y (mgr.25358) 566 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:26.590 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:51:27.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:27 smithi100 bash[80199]: cluster 2024-07-28T19:51:26.129087+0000 mgr.y (mgr.25358) 567 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:27.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:27 smithi100 bash[82644]: cluster 2024-07-28T19:51:26.129087+0000 mgr.y (mgr.25358) 567 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:27.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:27 smithi138 bash[67894]: cluster 2024-07-28T19:51:26.129087+0000 mgr.y (mgr.25358) 567 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:29 smithi100 bash[80199]: cluster 2024-07-28T19:51:28.129854+0000 mgr.y (mgr.25358) 568 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:29.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:29 smithi100 bash[82644]: cluster 2024-07-28T19:51:28.129854+0000 mgr.y (mgr.25358) 568 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:29.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:29 smithi138 bash[67894]: cluster 2024-07-28T19:51:28.129854+0000 mgr.y (mgr.25358) 568 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:31.370 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:51:31.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:31 smithi100 bash[80199]: cluster 2024-07-28T19:51:30.130833+0000 mgr.y (mgr.25358) 569 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:31.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:31 smithi100 bash[82644]: cluster 2024-07-28T19:51:30.130833+0000 mgr.y (mgr.25358) 569 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:31.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:31 smithi138 bash[67894]: cluster 2024-07-28T19:51:30.130833+0000 mgr.y (mgr.25358) 569 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:32.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:51:32 smithi138 bash[60615]: ts=2024-07-28T19:51:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:51:33.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:51:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:51:32] "GET /metrics HTTP/1.1" 200 34267 "" "Prometheus/2.43.0" 2024-07-28T19:51:33.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:33 smithi100 bash[80199]: cluster 2024-07-28T19:51:32.131583+0000 mgr.y (mgr.25358) 570 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:33.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:33 smithi100 bash[82644]: cluster 2024-07-28T19:51:32.131583+0000 mgr.y (mgr.25358) 570 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:33.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:33 smithi138 bash[67894]: cluster 2024-07-28T19:51:32.131583+0000 mgr.y (mgr.25358) 570 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:34.809 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:51:34.809 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:51:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:35 smithi100 bash[80199]: cluster 2024-07-28T19:51:34.132502+0000 mgr.y (mgr.25358) 571 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:35.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:35 smithi100 bash[80199]: audit 2024-07-28T19:51:34.811147+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.100:0/3915763329' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:35 smithi100 bash[82644]: cluster 2024-07-28T19:51:34.132502+0000 mgr.y (mgr.25358) 571 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:35 smithi100 bash[82644]: audit 2024-07-28T19:51:34.811147+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.100:0/3915763329' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:35.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:35 smithi138 bash[67894]: cluster 2024-07-28T19:51:34.132502+0000 mgr.y (mgr.25358) 571 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:35.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:35 smithi138 bash[67894]: audit 2024-07-28T19:51:34.811147+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.100:0/3915763329' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:37.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:37 smithi100 bash[80199]: cluster 2024-07-28T19:51:36.133402+0000 mgr.y (mgr.25358) 572 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:37.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:37 smithi100 bash[82644]: cluster 2024-07-28T19:51:36.133402+0000 mgr.y (mgr.25358) 572 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:37 smithi138 bash[67894]: cluster 2024-07-28T19:51:36.133402+0000 mgr.y (mgr.25358) 572 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:38.457 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:51:38.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:38 smithi100 bash[82644]: audit 2024-07-28T19:51:37.648859+0000 mon.a (mon.0) 1571 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:38.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:38 smithi100 bash[80199]: audit 2024-07-28T19:51:37.648859+0000 mon.a (mon.0) 1571 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:38 smithi138 bash[67894]: audit 2024-07-28T19:51:37.648859+0000 mon.a (mon.0) 1571 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:39.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:39 smithi100 bash[80199]: cluster 2024-07-28T19:51:38.134161+0000 mgr.y (mgr.25358) 573 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:39.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:39 smithi100 bash[82644]: cluster 2024-07-28T19:51:38.134161+0000 mgr.y (mgr.25358) 573 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:39 smithi138 bash[67894]: cluster 2024-07-28T19:51:38.134161+0000 mgr.y (mgr.25358) 573 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:41.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:41 smithi100 bash[80199]: cluster 2024-07-28T19:51:40.135157+0000 mgr.y (mgr.25358) 574 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:41.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:41 smithi100 bash[82644]: cluster 2024-07-28T19:51:40.135157+0000 mgr.y (mgr.25358) 574 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:41.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:41 smithi138 bash[67894]: cluster 2024-07-28T19:51:40.135157+0000 mgr.y (mgr.25358) 574 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:42.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:51:42 smithi138 bash[60615]: ts=2024-07-28T19:51:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:51:43.247 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:51:43.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:51:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:51:42] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-07-28T19:51:43.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:43 smithi100 bash[80199]: cluster 2024-07-28T19:51:42.135876+0000 mgr.y (mgr.25358) 575 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:43.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:43 smithi100 bash[82644]: cluster 2024-07-28T19:51:42.135876+0000 mgr.y (mgr.25358) 575 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:43.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:43 smithi138 bash[67894]: cluster 2024-07-28T19:51:42.135876+0000 mgr.y (mgr.25358) 575 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:45.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:45 smithi100 bash[80199]: cluster 2024-07-28T19:51:44.136812+0000 mgr.y (mgr.25358) 576 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:45.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:45 smithi100 bash[82644]: cluster 2024-07-28T19:51:44.136812+0000 mgr.y (mgr.25358) 576 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:45.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:45 smithi138 bash[67894]: cluster 2024-07-28T19:51:44.136812+0000 mgr.y (mgr.25358) 576 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:46.795 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:51:46.795 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:51:47.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:47 smithi100 bash[80199]: cluster 2024-07-28T19:51:46.137793+0000 mgr.y (mgr.25358) 577 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:47.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:47 smithi100 bash[80199]: audit 2024-07-28T19:51:46.797083+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.100:0/366879732' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:47.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:47 smithi100 bash[82644]: cluster 2024-07-28T19:51:46.137793+0000 mgr.y (mgr.25358) 577 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:47.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:47 smithi100 bash[82644]: audit 2024-07-28T19:51:46.797083+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.100:0/366879732' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:47 smithi138 bash[67894]: cluster 2024-07-28T19:51:46.137793+0000 mgr.y (mgr.25358) 577 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:47.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:47 smithi138 bash[67894]: audit 2024-07-28T19:51:46.797083+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.100:0/366879732' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:49.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:49 smithi100 bash[80199]: cluster 2024-07-28T19:51:48.138571+0000 mgr.y (mgr.25358) 578 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:49.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:49 smithi100 bash[82644]: cluster 2024-07-28T19:51:48.138571+0000 mgr.y (mgr.25358) 578 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:49.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:49 smithi138 bash[67894]: cluster 2024-07-28T19:51:48.138571+0000 mgr.y (mgr.25358) 578 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:50.486 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:51:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:51 smithi100 bash[80199]: cluster 2024-07-28T19:51:50.139565+0000 mgr.y (mgr.25358) 579 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:51 smithi100 bash[82644]: cluster 2024-07-28T19:51:50.139565+0000 mgr.y (mgr.25358) 579 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:51 smithi138 bash[67894]: cluster 2024-07-28T19:51:50.139565+0000 mgr.y (mgr.25358) 579 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:52.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:51:52 smithi138 bash[60615]: ts=2024-07-28T19:51:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:51:53.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:51:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:51:52] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-07-28T19:51:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:53 smithi100 bash[82644]: cluster 2024-07-28T19:51:52.140321+0000 mgr.y (mgr.25358) 580 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:53 smithi100 bash[82644]: audit 2024-07-28T19:51:52.649365+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:53 smithi100 bash[80199]: cluster 2024-07-28T19:51:52.140321+0000 mgr.y (mgr.25358) 580 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:53 smithi100 bash[80199]: audit 2024-07-28T19:51:52.649365+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:53 smithi138 bash[67894]: cluster 2024-07-28T19:51:52.140321+0000 mgr.y (mgr.25358) 580 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:53 smithi138 bash[67894]: audit 2024-07-28T19:51:52.649365+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:51:55.272 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:51:55.671 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:55 smithi100 bash[80199]: cluster 2024-07-28T19:51:54.141273+0000 mgr.y (mgr.25358) 581 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:55 smithi138 bash[67894]: cluster 2024-07-28T19:51:54.141273+0000 mgr.y (mgr.25358) 581 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:55.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:55 smithi100 bash[82644]: cluster 2024-07-28T19:51:54.141273+0000 mgr.y (mgr.25358) 581 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:57.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:57 smithi100 bash[80199]: cluster 2024-07-28T19:51:56.142086+0000 mgr.y (mgr.25358) 582 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:57.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:57 smithi100 bash[80199]: audit 2024-07-28T19:51:56.826502+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:51:57.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:57 smithi100 bash[82644]: cluster 2024-07-28T19:51:56.142086+0000 mgr.y (mgr.25358) 582 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:57.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:57 smithi100 bash[82644]: audit 2024-07-28T19:51:56.826502+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:51:57.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:57 smithi138 bash[67894]: cluster 2024-07-28T19:51:56.142086+0000 mgr.y (mgr.25358) 582 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:57.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:57 smithi138 bash[67894]: audit 2024-07-28T19:51:56.826502+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:51:58.954 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:51:58.954 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:51:59.678 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:59 smithi100 bash[82644]: cluster 2024-07-28T19:51:58.142808+0000 mgr.y (mgr.25358) 583 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:59.678 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:51:59 smithi100 bash[82644]: audit 2024-07-28T19:51:58.956749+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.100:0/314590634' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:59.678 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:59 smithi100 bash[80199]: cluster 2024-07-28T19:51:58.142808+0000 mgr.y (mgr.25358) 583 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:59.678 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:51:59 smithi100 bash[80199]: audit 2024-07-28T19:51:58.956749+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.100:0/314590634' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:51:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:59 smithi138 bash[67894]: cluster 2024-07-28T19:51:58.142808+0000 mgr.y (mgr.25358) 583 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:51:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:51:59 smithi138 bash[67894]: audit 2024-07-28T19:51:58.956749+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.100:0/314590634' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:01.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:01 smithi100 bash[82644]: cluster 2024-07-28T19:52:00.143744+0000 mgr.y (mgr.25358) 584 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:01.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:01 smithi100 bash[80199]: cluster 2024-07-28T19:52:00.143744+0000 mgr.y (mgr.25358) 584 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:01.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:01 smithi138 bash[67894]: cluster 2024-07-28T19:52:00.143744+0000 mgr.y (mgr.25358) 584 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:02.680 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:52:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:52:02 smithi138 bash[60615]: ts=2024-07-28T19:52:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:52:02.997 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:52:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:52:02] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-07-28T19:52:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[80199]: cluster 2024-07-28T19:52:02.144531+0000 mgr.y (mgr.25358) 585 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[80199]: audit 2024-07-28T19:52:02.595162+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[80199]: audit 2024-07-28T19:52:02.600304+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[80199]: audit 2024-07-28T19:52:02.889633+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[80199]: audit 2024-07-28T19:52:02.894467+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[80199]: audit 2024-07-28T19:52:03.305120+0000 mon.a (mon.0) 1578 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[80199]: audit 2024-07-28T19:52:03.306359+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[80199]: audit 2024-07-28T19:52:03.312315+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[82644]: cluster 2024-07-28T19:52:02.144531+0000 mgr.y (mgr.25358) 585 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[82644]: audit 2024-07-28T19:52:02.595162+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[82644]: audit 2024-07-28T19:52:02.600304+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[82644]: audit 2024-07-28T19:52:02.889633+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[82644]: audit 2024-07-28T19:52:02.894467+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[82644]: audit 2024-07-28T19:52:03.305120+0000 mon.a (mon.0) 1578 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[82644]: audit 2024-07-28T19:52:03.306359+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:52:03.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:03 smithi100 bash[82644]: audit 2024-07-28T19:52:03.312315+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:03 smithi138 bash[67894]: cluster 2024-07-28T19:52:02.144531+0000 mgr.y (mgr.25358) 585 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:03.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:03 smithi138 bash[67894]: audit 2024-07-28T19:52:02.595162+0000 mon.a (mon.0) 1574 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:03 smithi138 bash[67894]: audit 2024-07-28T19:52:02.600304+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:03 smithi138 bash[67894]: audit 2024-07-28T19:52:02.889633+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:03 smithi138 bash[67894]: audit 2024-07-28T19:52:02.894467+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:03.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:03 smithi138 bash[67894]: audit 2024-07-28T19:52:03.305120+0000 mon.a (mon.0) 1578 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:52:03.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:03 smithi138 bash[67894]: audit 2024-07-28T19:52:03.306359+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:52:03.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:03 smithi138 bash[67894]: audit 2024-07-28T19:52:03.312315+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:52:05.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:05 smithi100 bash[80199]: cluster 2024-07-28T19:52:04.145570+0000 mgr.y (mgr.25358) 586 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:05.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:05 smithi100 bash[82644]: cluster 2024-07-28T19:52:04.145570+0000 mgr.y (mgr.25358) 586 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:05.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:05 smithi138 bash[67894]: cluster 2024-07-28T19:52:04.145570+0000 mgr.y (mgr.25358) 586 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:07.462 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:52:07.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:07 smithi100 bash[80199]: cluster 2024-07-28T19:52:06.146577+0000 mgr.y (mgr.25358) 587 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:07.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:07 smithi100 bash[82644]: cluster 2024-07-28T19:52:06.146577+0000 mgr.y (mgr.25358) 587 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:07.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:07 smithi138 bash[67894]: cluster 2024-07-28T19:52:06.146577+0000 mgr.y (mgr.25358) 587 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:08.716 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:08 smithi100 bash[80199]: audit 2024-07-28T19:52:07.649819+0000 mon.a (mon.0) 1581 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:08.716 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:08 smithi100 bash[82644]: audit 2024-07-28T19:52:07.649819+0000 mon.a (mon.0) 1581 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:08.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:08 smithi138 bash[67894]: audit 2024-07-28T19:52:07.649819+0000 mon.a (mon.0) 1581 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:09.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:09 smithi100 bash[80199]: cluster 2024-07-28T19:52:08.147365+0000 mgr.y (mgr.25358) 588 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:09.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:09 smithi100 bash[82644]: cluster 2024-07-28T19:52:08.147365+0000 mgr.y (mgr.25358) 588 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:09.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:09 smithi138 bash[67894]: cluster 2024-07-28T19:52:08.147365+0000 mgr.y (mgr.25358) 588 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:11.089 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:52:11.089 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:52:11.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:11 smithi100 bash[82644]: cluster 2024-07-28T19:52:10.148424+0000 mgr.y (mgr.25358) 589 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:11.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:11 smithi100 bash[82644]: audit 2024-07-28T19:52:11.091276+0000 mon.a (mon.0) 1582 : audit [DBG] from='client.? 172.21.15.100:0/3278209901' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:11.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:11 smithi100 bash[80199]: cluster 2024-07-28T19:52:10.148424+0000 mgr.y (mgr.25358) 589 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:11.751 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:11 smithi100 bash[80199]: audit 2024-07-28T19:52:11.091276+0000 mon.a (mon.0) 1582 : audit [DBG] from='client.? 172.21.15.100:0/3278209901' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:11.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:11 smithi138 bash[67894]: cluster 2024-07-28T19:52:10.148424+0000 mgr.y (mgr.25358) 589 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:11.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:11 smithi138 bash[67894]: audit 2024-07-28T19:52:11.091276+0000 mon.a (mon.0) 1582 : audit [DBG] from='client.? 172.21.15.100:0/3278209901' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:12.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:52:12 smithi138 bash[60615]: ts=2024-07-28T19:52:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:52:13.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:52:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:52:12] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-07-28T19:52:13.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:13 smithi100 bash[82644]: cluster 2024-07-28T19:52:12.149299+0000 mgr.y (mgr.25358) 590 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:13.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:13 smithi100 bash[80199]: cluster 2024-07-28T19:52:12.149299+0000 mgr.y (mgr.25358) 590 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:13.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:13 smithi138 bash[67894]: cluster 2024-07-28T19:52:12.149299+0000 mgr.y (mgr.25358) 590 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:14.771 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:52:15.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:15 smithi100 bash[80199]: cluster 2024-07-28T19:52:14.150257+0000 mgr.y (mgr.25358) 591 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:15.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:15 smithi100 bash[82644]: cluster 2024-07-28T19:52:14.150257+0000 mgr.y (mgr.25358) 591 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:15.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:15 smithi138 bash[67894]: cluster 2024-07-28T19:52:14.150257+0000 mgr.y (mgr.25358) 591 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:17.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:17 smithi100 bash[80199]: cluster 2024-07-28T19:52:16.151164+0000 mgr.y (mgr.25358) 592 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:17.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:17 smithi100 bash[82644]: cluster 2024-07-28T19:52:16.151164+0000 mgr.y (mgr.25358) 592 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:17.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:17 smithi138 bash[67894]: cluster 2024-07-28T19:52:16.151164+0000 mgr.y (mgr.25358) 592 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:19.570 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:52:19.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:19 smithi100 bash[80199]: cluster 2024-07-28T19:52:18.151904+0000 mgr.y (mgr.25358) 593 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:19.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:19 smithi100 bash[82644]: cluster 2024-07-28T19:52:18.151904+0000 mgr.y (mgr.25358) 593 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:19.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:19 smithi138 bash[67894]: cluster 2024-07-28T19:52:18.151904+0000 mgr.y (mgr.25358) 593 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:21.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:21 smithi100 bash[80199]: cluster 2024-07-28T19:52:20.152896+0000 mgr.y (mgr.25358) 594 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:21.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:21 smithi100 bash[82644]: cluster 2024-07-28T19:52:20.152896+0000 mgr.y (mgr.25358) 594 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:21 smithi138 bash[67894]: cluster 2024-07-28T19:52:20.152896+0000 mgr.y (mgr.25358) 594 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:22.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:52:22 smithi138 bash[60615]: ts=2024-07-28T19:52:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:52:22.997 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:52:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:52:22] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-07-28T19:52:23.049 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:52:23.049 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:52:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:23 smithi100 bash[80199]: cluster 2024-07-28T19:52:22.153629+0000 mgr.y (mgr.25358) 595 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:23 smithi100 bash[80199]: audit 2024-07-28T19:52:22.650121+0000 mon.a (mon.0) 1583 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:23 smithi100 bash[80199]: audit 2024-07-28T19:52:23.051435+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.100:0/427765873' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:23 smithi100 bash[82644]: cluster 2024-07-28T19:52:22.153629+0000 mgr.y (mgr.25358) 595 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:23 smithi100 bash[82644]: audit 2024-07-28T19:52:22.650121+0000 mon.a (mon.0) 1583 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:23.749 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:23 smithi100 bash[82644]: audit 2024-07-28T19:52:23.051435+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.100:0/427765873' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:23.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:23 smithi138 bash[67894]: cluster 2024-07-28T19:52:22.153629+0000 mgr.y (mgr.25358) 595 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:23.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:23 smithi138 bash[67894]: audit 2024-07-28T19:52:22.650121+0000 mon.a (mon.0) 1583 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:23 smithi138 bash[67894]: audit 2024-07-28T19:52:23.051435+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.100:0/427765873' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:25 smithi100 bash[80199]: cluster 2024-07-28T19:52:24.154547+0000 mgr.y (mgr.25358) 596 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:25 smithi100 bash[82644]: cluster 2024-07-28T19:52:24.154547+0000 mgr.y (mgr.25358) 596 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:25 smithi138 bash[67894]: cluster 2024-07-28T19:52:24.154547+0000 mgr.y (mgr.25358) 596 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:26.794 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:52:27.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:27 smithi100 bash[80199]: cluster 2024-07-28T19:52:26.155424+0000 mgr.y (mgr.25358) 597 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:27.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:27 smithi100 bash[82644]: cluster 2024-07-28T19:52:26.155424+0000 mgr.y (mgr.25358) 597 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:27.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:27 smithi138 bash[67894]: cluster 2024-07-28T19:52:26.155424+0000 mgr.y (mgr.25358) 597 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:29.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:29 smithi100 bash[80199]: cluster 2024-07-28T19:52:28.156164+0000 mgr.y (mgr.25358) 598 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:29.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:29 smithi100 bash[82644]: cluster 2024-07-28T19:52:28.156164+0000 mgr.y (mgr.25358) 598 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:29.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:29 smithi138 bash[67894]: cluster 2024-07-28T19:52:28.156164+0000 mgr.y (mgr.25358) 598 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:31.590 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:52:31.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:31 smithi138 bash[67894]: cluster 2024-07-28T19:52:30.157161+0000 mgr.y (mgr.25358) 599 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:31.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:31 smithi100 bash[80199]: cluster 2024-07-28T19:52:30.157161+0000 mgr.y (mgr.25358) 599 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:31.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:31 smithi100 bash[82644]: cluster 2024-07-28T19:52:30.157161+0000 mgr.y (mgr.25358) 599 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:32.772 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:32 smithi100 bash[80199]: cluster 2024-07-28T19:52:32.157873+0000 mgr.y (mgr.25358) 600 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:32.772 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:32 smithi100 bash[82644]: cluster 2024-07-28T19:52:32.157873+0000 mgr.y (mgr.25358) 600 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:32.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:32 smithi138 bash[67894]: cluster 2024-07-28T19:52:32.157873+0000 mgr.y (mgr.25358) 600 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:32.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:52:32 smithi138 bash[60615]: ts=2024-07-28T19:52:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:52:33.088 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:52:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:52:32] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-07-28T19:52:35.112 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:52:35.112 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:52:35.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:35 smithi100 bash[80199]: cluster 2024-07-28T19:52:34.158781+0000 mgr.y (mgr.25358) 601 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:35.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:35 smithi100 bash[80199]: audit 2024-07-28T19:52:35.114895+0000 mon.a (mon.0) 1585 : audit [DBG] from='client.? 172.21.15.100:0/3476007971' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:35.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:35 smithi100 bash[82644]: cluster 2024-07-28T19:52:34.158781+0000 mgr.y (mgr.25358) 601 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:35.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:35 smithi100 bash[82644]: audit 2024-07-28T19:52:35.114895+0000 mon.a (mon.0) 1585 : audit [DBG] from='client.? 172.21.15.100:0/3476007971' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:35 smithi138 bash[67894]: cluster 2024-07-28T19:52:34.158781+0000 mgr.y (mgr.25358) 601 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:35 smithi138 bash[67894]: audit 2024-07-28T19:52:35.114895+0000 mon.a (mon.0) 1585 : audit [DBG] from='client.? 172.21.15.100:0/3476007971' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:37.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:37 smithi100 bash[80199]: cluster 2024-07-28T19:52:36.159678+0000 mgr.y (mgr.25358) 602 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:37.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:37 smithi100 bash[82644]: cluster 2024-07-28T19:52:36.159678+0000 mgr.y (mgr.25358) 602 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:37.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:37 smithi138 bash[67894]: cluster 2024-07-28T19:52:36.159678+0000 mgr.y (mgr.25358) 602 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:38.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:38 smithi100 bash[80199]: audit 2024-07-28T19:52:37.650608+0000 mon.a (mon.0) 1586 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:38.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:38 smithi100 bash[82644]: audit 2024-07-28T19:52:37.650608+0000 mon.a (mon.0) 1586 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:38.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:38 smithi138 bash[67894]: audit 2024-07-28T19:52:37.650608+0000 mon.a (mon.0) 1586 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:38.837 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:52:39.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:39 smithi100 bash[80199]: cluster 2024-07-28T19:52:38.160387+0000 mgr.y (mgr.25358) 603 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:39.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:39 smithi100 bash[82644]: cluster 2024-07-28T19:52:38.160387+0000 mgr.y (mgr.25358) 603 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:39.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:39 smithi138 bash[67894]: cluster 2024-07-28T19:52:38.160387+0000 mgr.y (mgr.25358) 603 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:41.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:41 smithi100 bash[80199]: cluster 2024-07-28T19:52:40.161429+0000 mgr.y (mgr.25358) 604 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:41.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:41 smithi100 bash[82644]: cluster 2024-07-28T19:52:40.161429+0000 mgr.y (mgr.25358) 604 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:41.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:41 smithi138 bash[67894]: cluster 2024-07-28T19:52:40.161429+0000 mgr.y (mgr.25358) 604 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:42.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:52:42 smithi138 bash[60615]: ts=2024-07-28T19:52:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:52:43.236 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:52:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:52:42] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-07-28T19:52:43.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:43 smithi100 bash[80199]: cluster 2024-07-28T19:52:42.162158+0000 mgr.y (mgr.25358) 605 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:43.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:43 smithi100 bash[82644]: cluster 2024-07-28T19:52:42.162158+0000 mgr.y (mgr.25358) 605 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:43.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:43 smithi138 bash[67894]: cluster 2024-07-28T19:52:42.162158+0000 mgr.y (mgr.25358) 605 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:43.631 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:52:45.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:45 smithi100 bash[82644]: cluster 2024-07-28T19:52:44.163071+0000 mgr.y (mgr.25358) 606 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:45.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:45 smithi100 bash[80199]: cluster 2024-07-28T19:52:44.163071+0000 mgr.y (mgr.25358) 606 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:45.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:45 smithi138 bash[67894]: cluster 2024-07-28T19:52:44.163071+0000 mgr.y (mgr.25358) 606 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:47.241 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:52:47.242 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:52:47.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:47 smithi100 bash[80199]: cluster 2024-07-28T19:52:46.163958+0000 mgr.y (mgr.25358) 607 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:47.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:47 smithi100 bash[80199]: audit 2024-07-28T19:52:47.244042+0000 mon.a (mon.0) 1587 : audit [DBG] from='client.? 172.21.15.100:0/1904984367' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:47.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:47 smithi100 bash[82644]: cluster 2024-07-28T19:52:46.163958+0000 mgr.y (mgr.25358) 607 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:47.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:47 smithi100 bash[82644]: audit 2024-07-28T19:52:47.244042+0000 mon.a (mon.0) 1587 : audit [DBG] from='client.? 172.21.15.100:0/1904984367' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:47.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:47 smithi138 bash[67894]: cluster 2024-07-28T19:52:46.163958+0000 mgr.y (mgr.25358) 607 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:47.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:47 smithi138 bash[67894]: audit 2024-07-28T19:52:47.244042+0000 mon.a (mon.0) 1587 : audit [DBG] from='client.? 172.21.15.100:0/1904984367' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:49.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:49 smithi138 bash[67894]: cluster 2024-07-28T19:52:48.164707+0000 mgr.y (mgr.25358) 608 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:49.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:49 smithi100 bash[80199]: cluster 2024-07-28T19:52:48.164707+0000 mgr.y (mgr.25358) 608 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:49.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:49 smithi100 bash[82644]: cluster 2024-07-28T19:52:48.164707+0000 mgr.y (mgr.25358) 608 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:50.972 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:52:51.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:51 smithi138 bash[67894]: cluster 2024-07-28T19:52:50.165669+0000 mgr.y (mgr.25358) 609 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:51.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:51 smithi100 bash[80199]: cluster 2024-07-28T19:52:50.165669+0000 mgr.y (mgr.25358) 609 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:51.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:51 smithi100 bash[82644]: cluster 2024-07-28T19:52:50.165669+0000 mgr.y (mgr.25358) 609 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:52.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:52:52 smithi138 bash[60615]: ts=2024-07-28T19:52:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:52:53.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:52:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:52:52] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-07-28T19:52:53.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:53 smithi138 bash[67894]: cluster 2024-07-28T19:52:52.166378+0000 mgr.y (mgr.25358) 610 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:53.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:53 smithi138 bash[67894]: audit 2024-07-28T19:52:52.651019+0000 mon.a (mon.0) 1588 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:53 smithi100 bash[80199]: cluster 2024-07-28T19:52:52.166378+0000 mgr.y (mgr.25358) 610 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:53.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:53 smithi100 bash[80199]: audit 2024-07-28T19:52:52.651019+0000 mon.a (mon.0) 1588 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:53 smithi100 bash[82644]: cluster 2024-07-28T19:52:52.166378+0000 mgr.y (mgr.25358) 610 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:53 smithi100 bash[82644]: audit 2024-07-28T19:52:52.651019+0000 mon.a (mon.0) 1588 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:52:55.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:55 smithi138 bash[67894]: cluster 2024-07-28T19:52:54.167271+0000 mgr.y (mgr.25358) 611 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:55.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:55 smithi100 bash[80199]: cluster 2024-07-28T19:52:54.167271+0000 mgr.y (mgr.25358) 611 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:55.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:55 smithi100 bash[82644]: cluster 2024-07-28T19:52:54.167271+0000 mgr.y (mgr.25358) 611 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:55.756 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:52:57.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:57 smithi100 bash[80199]: cluster 2024-07-28T19:52:56.168122+0000 mgr.y (mgr.25358) 612 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:57.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:57 smithi100 bash[82644]: cluster 2024-07-28T19:52:56.168122+0000 mgr.y (mgr.25358) 612 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:57.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:57 smithi138 bash[67894]: cluster 2024-07-28T19:52:56.168122+0000 mgr.y (mgr.25358) 612 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:59.150 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:52:59.150 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:52:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:59 smithi100 bash[80199]: cluster 2024-07-28T19:52:58.168837+0000 mgr.y (mgr.25358) 613 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:59.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:52:59 smithi100 bash[80199]: audit 2024-07-28T19:52:59.152350+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.100:0/100281576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:59 smithi100 bash[82644]: cluster 2024-07-28T19:52:58.168837+0000 mgr.y (mgr.25358) 613 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:59.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:52:59 smithi100 bash[82644]: audit 2024-07-28T19:52:59.152350+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.100:0/100281576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:52:59.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:59 smithi138 bash[67894]: cluster 2024-07-28T19:52:58.168837+0000 mgr.y (mgr.25358) 613 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:52:59.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:52:59 smithi138 bash[67894]: audit 2024-07-28T19:52:59.152350+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.100:0/100281576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:01.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:01 smithi138 bash[67894]: cluster 2024-07-28T19:53:00.169790+0000 mgr.y (mgr.25358) 614 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:01.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:01 smithi100 bash[80199]: cluster 2024-07-28T19:53:00.169790+0000 mgr.y (mgr.25358) 614 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:01.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:01 smithi100 bash[82644]: cluster 2024-07-28T19:53:00.169790+0000 mgr.y (mgr.25358) 614 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:53:02 smithi138 bash[60615]: ts=2024-07-28T19:53:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:53:02.855 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:53:03.248 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:53:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:53:02] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-07-28T19:53:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:03 smithi138 bash[67894]: cluster 2024-07-28T19:53:02.170498+0000 mgr.y (mgr.25358) 615 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:03.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:03 smithi100 bash[80199]: cluster 2024-07-28T19:53:02.170498+0000 mgr.y (mgr.25358) 615 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:03.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:03 smithi100 bash[82644]: cluster 2024-07-28T19:53:02.170498+0000 mgr.y (mgr.25358) 615 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:04.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:04 smithi138 bash[67894]: audit 2024-07-28T19:53:03.385195+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:53:04.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:04 smithi100 bash[80199]: audit 2024-07-28T19:53:03.385195+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:53:04.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:04 smithi100 bash[82644]: audit 2024-07-28T19:53:03.385195+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:53:05.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:05 smithi138 bash[67894]: cluster 2024-07-28T19:53:04.171380+0000 mgr.y (mgr.25358) 616 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:05.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:05 smithi100 bash[80199]: cluster 2024-07-28T19:53:04.171380+0000 mgr.y (mgr.25358) 616 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:05.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:05 smithi100 bash[82644]: cluster 2024-07-28T19:53:04.171380+0000 mgr.y (mgr.25358) 616 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:07.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:07 smithi138 bash[67894]: cluster 2024-07-28T19:53:06.172283+0000 mgr.y (mgr.25358) 617 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:07.635 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:53:07.728 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:07 smithi100 bash[80199]: cluster 2024-07-28T19:53:06.172283+0000 mgr.y (mgr.25358) 617 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:07.729 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:07 smithi100 bash[82644]: cluster 2024-07-28T19:53:06.172283+0000 mgr.y (mgr.25358) 617 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:08 smithi138 bash[67894]: audit 2024-07-28T19:53:07.651145+0000 mon.a (mon.0) 1591 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:08 smithi138 bash[67894]: audit 2024-07-28T19:53:08.270609+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:08.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:08 smithi138 bash[67894]: audit 2024-07-28T19:53:08.277170+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:08.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:08 smithi100 bash[80199]: audit 2024-07-28T19:53:07.651145+0000 mon.a (mon.0) 1591 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:08 smithi100 bash[80199]: audit 2024-07-28T19:53:08.270609+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:08.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:08 smithi100 bash[80199]: audit 2024-07-28T19:53:08.277170+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:08.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:08 smithi100 bash[82644]: audit 2024-07-28T19:53:07.651145+0000 mon.a (mon.0) 1591 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:08.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:08 smithi100 bash[82644]: audit 2024-07-28T19:53:08.270609+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:08.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:08 smithi100 bash[82644]: audit 2024-07-28T19:53:08.277170+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:09 smithi138 bash[67894]: cluster 2024-07-28T19:53:08.172738+0000 mgr.y (mgr.25358) 618 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:09 smithi138 bash[67894]: audit 2024-07-28T19:53:09.206231+0000 mon.a (mon.0) 1594 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:09.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:09 smithi138 bash[67894]: audit 2024-07-28T19:53:09.216288+0000 mon.a (mon.0) 1595 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:09.722 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:09 smithi100 bash[82644]: cluster 2024-07-28T19:53:08.172738+0000 mgr.y (mgr.25358) 618 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:09.722 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:09 smithi100 bash[82644]: audit 2024-07-28T19:53:09.206231+0000 mon.a (mon.0) 1594 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:09.722 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:09 smithi100 bash[82644]: audit 2024-07-28T19:53:09.216288+0000 mon.a (mon.0) 1595 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:09.722 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:09 smithi100 bash[80199]: cluster 2024-07-28T19:53:08.172738+0000 mgr.y (mgr.25358) 618 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:09.722 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:09 smithi100 bash[80199]: audit 2024-07-28T19:53:09.206231+0000 mon.a (mon.0) 1594 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:09.723 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:09 smithi100 bash[80199]: audit 2024-07-28T19:53:09.216288+0000 mon.a (mon.0) 1595 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:10.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:10 smithi138 bash[67894]: audit 2024-07-28T19:53:09.685757+0000 mon.a (mon.0) 1596 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:53:10.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:10 smithi138 bash[67894]: audit 2024-07-28T19:53:09.686955+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:53:10.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:10 smithi138 bash[67894]: audit 2024-07-28T19:53:09.694840+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:10.717 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:10 smithi100 bash[80199]: audit 2024-07-28T19:53:09.685757+0000 mon.a (mon.0) 1596 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:53:10.717 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:10 smithi100 bash[80199]: audit 2024-07-28T19:53:09.686955+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:53:10.717 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:10 smithi100 bash[80199]: audit 2024-07-28T19:53:09.694840+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:10.717 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:10 smithi100 bash[82644]: audit 2024-07-28T19:53:09.685757+0000 mon.a (mon.0) 1596 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:53:10.717 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:10 smithi100 bash[82644]: audit 2024-07-28T19:53:09.686955+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:53:10.717 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:10 smithi100 bash[82644]: audit 2024-07-28T19:53:09.694840+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:53:11.096 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:53:11.096 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:53:11.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:11 smithi100 bash[80199]: cluster 2024-07-28T19:53:10.173804+0000 mgr.y (mgr.25358) 619 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:11.498 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:11 smithi100 bash[80199]: audit 2024-07-28T19:53:11.097292+0000 mon.a (mon.0) 1599 : audit [DBG] from='client.? 172.21.15.100:0/4230291915' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:11.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:11 smithi100 bash[82644]: cluster 2024-07-28T19:53:10.173804+0000 mgr.y (mgr.25358) 619 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:11.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:11 smithi100 bash[82644]: audit 2024-07-28T19:53:11.097292+0000 mon.a (mon.0) 1599 : audit [DBG] from='client.? 172.21.15.100:0/4230291915' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:11.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:11 smithi138 bash[67894]: cluster 2024-07-28T19:53:10.173804+0000 mgr.y (mgr.25358) 619 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:11.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:11 smithi138 bash[67894]: audit 2024-07-28T19:53:11.097292+0000 mon.a (mon.0) 1599 : audit [DBG] from='client.? 172.21.15.100:0/4230291915' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:12.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:53:12 smithi138 bash[60615]: ts=2024-07-28T19:53:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:53:13.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:53:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:53:12] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-07-28T19:53:13.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:13 smithi100 bash[80199]: cluster 2024-07-28T19:53:12.174661+0000 mgr.y (mgr.25358) 620 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:13.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:13 smithi100 bash[82644]: cluster 2024-07-28T19:53:12.174661+0000 mgr.y (mgr.25358) 620 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:13.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:13 smithi138 bash[67894]: cluster 2024-07-28T19:53:12.174661+0000 mgr.y (mgr.25358) 620 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:14.888 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:53:15.750 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:15 smithi100 bash[80199]: cluster 2024-07-28T19:53:14.175574+0000 mgr.y (mgr.25358) 621 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:15.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:15 smithi100 bash[82644]: cluster 2024-07-28T19:53:14.175574+0000 mgr.y (mgr.25358) 621 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:15.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:15 smithi138 bash[67894]: cluster 2024-07-28T19:53:14.175574+0000 mgr.y (mgr.25358) 621 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:17.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:17 smithi100 bash[80199]: cluster 2024-07-28T19:53:16.176419+0000 mgr.y (mgr.25358) 622 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:17.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:17 smithi100 bash[82644]: cluster 2024-07-28T19:53:16.176419+0000 mgr.y (mgr.25358) 622 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:17.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:17 smithi138 bash[67894]: cluster 2024-07-28T19:53:16.176419+0000 mgr.y (mgr.25358) 622 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:19.675 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:53:19.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:19 smithi100 bash[80199]: cluster 2024-07-28T19:53:18.177230+0000 mgr.y (mgr.25358) 623 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:19.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:19 smithi100 bash[82644]: cluster 2024-07-28T19:53:18.177230+0000 mgr.y (mgr.25358) 623 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:19.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:19 smithi138 bash[67894]: cluster 2024-07-28T19:53:18.177230+0000 mgr.y (mgr.25358) 623 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:21.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:21 smithi100 bash[80199]: cluster 2024-07-28T19:53:20.178176+0000 mgr.y (mgr.25358) 624 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:21.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:21 smithi100 bash[82644]: cluster 2024-07-28T19:53:20.178176+0000 mgr.y (mgr.25358) 624 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:21 smithi138 bash[67894]: cluster 2024-07-28T19:53:20.178176+0000 mgr.y (mgr.25358) 624 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:22.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:53:22 smithi138 bash[60615]: ts=2024-07-28T19:53:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:53:23.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:53:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:53:22] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-07-28T19:53:23.292 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:53:23.292 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:53:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:23 smithi100 bash[80199]: cluster 2024-07-28T19:53:22.178968+0000 mgr.y (mgr.25358) 625 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:23 smithi100 bash[80199]: audit 2024-07-28T19:53:22.651635+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:23.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:23 smithi100 bash[80199]: audit 2024-07-28T19:53:23.294769+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.100:0/78728824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:23 smithi100 bash[82644]: cluster 2024-07-28T19:53:22.178968+0000 mgr.y (mgr.25358) 625 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:23 smithi100 bash[82644]: audit 2024-07-28T19:53:22.651635+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:23 smithi100 bash[82644]: audit 2024-07-28T19:53:23.294769+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.100:0/78728824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:23.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:23 smithi138 bash[67894]: cluster 2024-07-28T19:53:22.178968+0000 mgr.y (mgr.25358) 625 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:23.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:23 smithi138 bash[67894]: audit 2024-07-28T19:53:22.651635+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:23.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:23 smithi138 bash[67894]: audit 2024-07-28T19:53:23.294769+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.100:0/78728824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:25.748 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:25 smithi100 bash[80199]: cluster 2024-07-28T19:53:24.179872+0000 mgr.y (mgr.25358) 626 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:25 smithi100 bash[82644]: cluster 2024-07-28T19:53:24.179872+0000 mgr.y (mgr.25358) 626 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:25.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:25 smithi138 bash[67894]: cluster 2024-07-28T19:53:24.179872+0000 mgr.y (mgr.25358) 626 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:27.050 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:53:27.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:27 smithi100 bash[80199]: cluster 2024-07-28T19:53:26.180726+0000 mgr.y (mgr.25358) 627 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:27.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:27 smithi100 bash[82644]: cluster 2024-07-28T19:53:26.180726+0000 mgr.y (mgr.25358) 627 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:27.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:27 smithi138 bash[67894]: cluster 2024-07-28T19:53:26.180726+0000 mgr.y (mgr.25358) 627 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:29.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:29 smithi100 bash[80199]: cluster 2024-07-28T19:53:28.181458+0000 mgr.y (mgr.25358) 628 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:29.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:29 smithi100 bash[82644]: cluster 2024-07-28T19:53:28.181458+0000 mgr.y (mgr.25358) 628 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:29.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:29 smithi138 bash[67894]: cluster 2024-07-28T19:53:28.181458+0000 mgr.y (mgr.25358) 628 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:31.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:31 smithi100 bash[80199]: cluster 2024-07-28T19:53:30.182387+0000 mgr.y (mgr.25358) 629 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:31.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:31 smithi100 bash[82644]: cluster 2024-07-28T19:53:30.182387+0000 mgr.y (mgr.25358) 629 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:31.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:31 smithi138 bash[67894]: cluster 2024-07-28T19:53:30.182387+0000 mgr.y (mgr.25358) 629 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:31.834 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:53:32.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:53:32 smithi138 bash[60615]: ts=2024-07-28T19:53:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:53:33.166 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:53:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:53:32] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-07-28T19:53:33.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:33 smithi100 bash[80199]: cluster 2024-07-28T19:53:32.183096+0000 mgr.y (mgr.25358) 630 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:33.498 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:33 smithi100 bash[82644]: cluster 2024-07-28T19:53:32.183096+0000 mgr.y (mgr.25358) 630 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:33.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:33 smithi138 bash[67894]: cluster 2024-07-28T19:53:32.183096+0000 mgr.y (mgr.25358) 630 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:35.333 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:53:35.333 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:53:35.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:35 smithi100 bash[80199]: cluster 2024-07-28T19:53:34.183985+0000 mgr.y (mgr.25358) 631 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:35.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:35 smithi100 bash[80199]: audit 2024-07-28T19:53:35.336259+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.100:0/2031666788' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:35 smithi100 bash[82644]: cluster 2024-07-28T19:53:34.183985+0000 mgr.y (mgr.25358) 631 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:35 smithi100 bash[82644]: audit 2024-07-28T19:53:35.336259+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.100:0/2031666788' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:35.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:35 smithi138 bash[67894]: cluster 2024-07-28T19:53:34.183985+0000 mgr.y (mgr.25358) 631 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:35.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:35 smithi138 bash[67894]: audit 2024-07-28T19:53:35.336259+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.100:0/2031666788' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:37.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:37 smithi100 bash[80199]: cluster 2024-07-28T19:53:36.184839+0000 mgr.y (mgr.25358) 632 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:37.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:37 smithi100 bash[82644]: cluster 2024-07-28T19:53:36.184839+0000 mgr.y (mgr.25358) 632 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:37 smithi138 bash[67894]: cluster 2024-07-28T19:53:36.184839+0000 mgr.y (mgr.25358) 632 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:38.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:38 smithi100 bash[80199]: audit 2024-07-28T19:53:37.652152+0000 mon.a (mon.0) 1602 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:38.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:38 smithi100 bash[82644]: audit 2024-07-28T19:53:37.652152+0000 mon.a (mon.0) 1602 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:38 smithi138 bash[67894]: audit 2024-07-28T19:53:37.652152+0000 mon.a (mon.0) 1602 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:39.081 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:53:39.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:39 smithi100 bash[80199]: cluster 2024-07-28T19:53:38.185546+0000 mgr.y (mgr.25358) 633 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:39.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:39 smithi100 bash[82644]: cluster 2024-07-28T19:53:38.185546+0000 mgr.y (mgr.25358) 633 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:39 smithi138 bash[67894]: cluster 2024-07-28T19:53:38.185546+0000 mgr.y (mgr.25358) 633 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:41.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:41 smithi100 bash[80199]: cluster 2024-07-28T19:53:40.186578+0000 mgr.y (mgr.25358) 634 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:41.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:41 smithi100 bash[82644]: cluster 2024-07-28T19:53:40.186578+0000 mgr.y (mgr.25358) 634 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:41.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:41 smithi138 bash[67894]: cluster 2024-07-28T19:53:40.186578+0000 mgr.y (mgr.25358) 634 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:42.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:53:42 smithi138 bash[60615]: ts=2024-07-28T19:53:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:53:43.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:53:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:53:42] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-07-28T19:53:43.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:43 smithi100 bash[80199]: cluster 2024-07-28T19:53:42.187405+0000 mgr.y (mgr.25358) 635 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:43.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:43 smithi100 bash[82644]: cluster 2024-07-28T19:53:42.187405+0000 mgr.y (mgr.25358) 635 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:43.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:43 smithi138 bash[67894]: cluster 2024-07-28T19:53:42.187405+0000 mgr.y (mgr.25358) 635 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:43.865 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:53:45.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:45 smithi100 bash[80199]: cluster 2024-07-28T19:53:44.188350+0000 mgr.y (mgr.25358) 636 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:45.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:45 smithi100 bash[82644]: cluster 2024-07-28T19:53:44.188350+0000 mgr.y (mgr.25358) 636 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:45.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:45 smithi138 bash[67894]: cluster 2024-07-28T19:53:44.188350+0000 mgr.y (mgr.25358) 636 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:47.420 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:53:47.420 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:53:47.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:47 smithi100 bash[80199]: cluster 2024-07-28T19:53:46.189294+0000 mgr.y (mgr.25358) 637 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:47.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:47 smithi100 bash[82644]: cluster 2024-07-28T19:53:46.189294+0000 mgr.y (mgr.25358) 637 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:47 smithi138 bash[67894]: cluster 2024-07-28T19:53:46.189294+0000 mgr.y (mgr.25358) 637 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:48.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:48 smithi100 bash[80199]: audit 2024-07-28T19:53:47.423141+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.100:0/469997438' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:48.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:48 smithi100 bash[82644]: audit 2024-07-28T19:53:47.423141+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.100:0/469997438' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:48.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:48 smithi138 bash[67894]: audit 2024-07-28T19:53:47.423141+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.100:0/469997438' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:49.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:49 smithi100 bash[80199]: cluster 2024-07-28T19:53:48.190079+0000 mgr.y (mgr.25358) 638 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:49.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:49 smithi100 bash[82644]: cluster 2024-07-28T19:53:48.190079+0000 mgr.y (mgr.25358) 638 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:49.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:49 smithi138 bash[67894]: cluster 2024-07-28T19:53:48.190079+0000 mgr.y (mgr.25358) 638 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:51.113 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:53:51.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:51 smithi100 bash[80199]: cluster 2024-07-28T19:53:50.191047+0000 mgr.y (mgr.25358) 639 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:51 smithi100 bash[82644]: cluster 2024-07-28T19:53:50.191047+0000 mgr.y (mgr.25358) 639 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:51 smithi138 bash[67894]: cluster 2024-07-28T19:53:50.191047+0000 mgr.y (mgr.25358) 639 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:52.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:53:52 smithi138 bash[60615]: ts=2024-07-28T19:53:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:53:53.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:53:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:53:52] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-07-28T19:53:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:53 smithi100 bash[80199]: cluster 2024-07-28T19:53:52.191773+0000 mgr.y (mgr.25358) 640 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:53 smithi100 bash[80199]: audit 2024-07-28T19:53:52.652647+0000 mon.a (mon.0) 1603 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:53 smithi100 bash[82644]: cluster 2024-07-28T19:53:52.191773+0000 mgr.y (mgr.25358) 640 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:53.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:53 smithi100 bash[82644]: audit 2024-07-28T19:53:52.652647+0000 mon.a (mon.0) 1603 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:53 smithi138 bash[67894]: cluster 2024-07-28T19:53:52.191773+0000 mgr.y (mgr.25358) 640 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:53 smithi138 bash[67894]: audit 2024-07-28T19:53:52.652647+0000 mon.a (mon.0) 1603 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:53:55.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:55 smithi100 bash[80199]: cluster 2024-07-28T19:53:54.192704+0000 mgr.y (mgr.25358) 641 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:55.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:55 smithi100 bash[82644]: cluster 2024-07-28T19:53:54.192704+0000 mgr.y (mgr.25358) 641 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:55 smithi138 bash[67894]: cluster 2024-07-28T19:53:54.192704+0000 mgr.y (mgr.25358) 641 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:55.897 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:53:57.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:57 smithi100 bash[80199]: cluster 2024-07-28T19:53:56.193644+0000 mgr.y (mgr.25358) 642 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:57.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:57 smithi100 bash[82644]: cluster 2024-07-28T19:53:56.193644+0000 mgr.y (mgr.25358) 642 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:57.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:57 smithi138 bash[67894]: cluster 2024-07-28T19:53:56.193644+0000 mgr.y (mgr.25358) 642 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:59.352 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:53:59.353 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:53:59.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:59 smithi100 bash[80199]: cluster 2024-07-28T19:53:58.194414+0000 mgr.y (mgr.25358) 643 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:59.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:53:59 smithi100 bash[80199]: audit 2024-07-28T19:53:59.355455+0000 mon.a (mon.0) 1604 : audit [DBG] from='client.? 172.21.15.100:0/3410733418' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:59.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:59 smithi100 bash[82644]: cluster 2024-07-28T19:53:58.194414+0000 mgr.y (mgr.25358) 643 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:59.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:53:59 smithi100 bash[82644]: audit 2024-07-28T19:53:59.355455+0000 mon.a (mon.0) 1604 : audit [DBG] from='client.? 172.21.15.100:0/3410733418' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:53:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:59 smithi138 bash[67894]: cluster 2024-07-28T19:53:58.194414+0000 mgr.y (mgr.25358) 643 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:53:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:53:59 smithi138 bash[67894]: audit 2024-07-28T19:53:59.355455+0000 mon.a (mon.0) 1604 : audit [DBG] from='client.? 172.21.15.100:0/3410733418' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:01.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:01 smithi100 bash[80199]: cluster 2024-07-28T19:54:00.195415+0000 mgr.y (mgr.25358) 644 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:01.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:01 smithi100 bash[82644]: cluster 2024-07-28T19:54:00.195415+0000 mgr.y (mgr.25358) 644 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:01.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:01 smithi138 bash[67894]: cluster 2024-07-28T19:54:00.195415+0000 mgr.y (mgr.25358) 644 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:54:02 smithi138 bash[60615]: ts=2024-07-28T19:54:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:54:03.054 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:54:03.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:54:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:54:02] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-07-28T19:54:03.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:03 smithi100 bash[80199]: cluster 2024-07-28T19:54:02.196132+0000 mgr.y (mgr.25358) 645 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:03.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:03 smithi100 bash[82644]: cluster 2024-07-28T19:54:02.196132+0000 mgr.y (mgr.25358) 645 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:03.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:03 smithi138 bash[67894]: cluster 2024-07-28T19:54:02.196132+0000 mgr.y (mgr.25358) 645 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:05.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:05 smithi100 bash[80199]: cluster 2024-07-28T19:54:04.197067+0000 mgr.y (mgr.25358) 646 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:05.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:05 smithi100 bash[82644]: cluster 2024-07-28T19:54:04.197067+0000 mgr.y (mgr.25358) 646 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:05.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:05 smithi138 bash[67894]: cluster 2024-07-28T19:54:04.197067+0000 mgr.y (mgr.25358) 646 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:07.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:07 smithi100 bash[80199]: cluster 2024-07-28T19:54:06.197935+0000 mgr.y (mgr.25358) 647 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:07.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:07 smithi100 bash[82644]: cluster 2024-07-28T19:54:06.197935+0000 mgr.y (mgr.25358) 647 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:07.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:07 smithi138 bash[67894]: cluster 2024-07-28T19:54:06.197935+0000 mgr.y (mgr.25358) 647 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:07.838 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:54:08.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:08 smithi100 bash[80199]: audit 2024-07-28T19:54:07.653066+0000 mon.a (mon.0) 1605 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:08.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:08 smithi100 bash[82644]: audit 2024-07-28T19:54:07.653066+0000 mon.a (mon.0) 1605 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:08.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:08 smithi138 bash[67894]: audit 2024-07-28T19:54:07.653066+0000 mon.a (mon.0) 1605 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:09.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:09 smithi100 bash[80199]: cluster 2024-07-28T19:54:08.198677+0000 mgr.y (mgr.25358) 648 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:09.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:09 smithi100 bash[82644]: cluster 2024-07-28T19:54:08.198677+0000 mgr.y (mgr.25358) 648 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:09.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:09 smithi138 bash[67894]: cluster 2024-07-28T19:54:08.198677+0000 mgr.y (mgr.25358) 648 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:10.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:10 smithi100 bash[80199]: audit 2024-07-28T19:54:09.773220+0000 mon.a (mon.0) 1606 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:54:10.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:10 smithi100 bash[82644]: audit 2024-07-28T19:54:09.773220+0000 mon.a (mon.0) 1606 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:54:10.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:10 smithi138 bash[67894]: audit 2024-07-28T19:54:09.773220+0000 mon.a (mon.0) 1606 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:54:11.308 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:54:11.308 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:54:11.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:11 smithi100 bash[80199]: cluster 2024-07-28T19:54:10.199664+0000 mgr.y (mgr.25358) 649 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:11.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:11 smithi100 bash[80199]: audit 2024-07-28T19:54:11.310896+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.100:0/2463027458' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:11.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:11 smithi100 bash[82644]: cluster 2024-07-28T19:54:10.199664+0000 mgr.y (mgr.25358) 649 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:11.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:11 smithi100 bash[82644]: audit 2024-07-28T19:54:11.310896+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.100:0/2463027458' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:11.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:11 smithi138 bash[67894]: cluster 2024-07-28T19:54:10.199664+0000 mgr.y (mgr.25358) 649 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:11.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:11 smithi138 bash[67894]: audit 2024-07-28T19:54:11.310896+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.100:0/2463027458' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:12.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:54:12 smithi138 bash[60615]: ts=2024-07-28T19:54:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:54:13.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:54:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:54:12] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-07-28T19:54:13.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:13 smithi100 bash[80199]: cluster 2024-07-28T19:54:12.200405+0000 mgr.y (mgr.25358) 650 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:13.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:13 smithi100 bash[82644]: cluster 2024-07-28T19:54:12.200405+0000 mgr.y (mgr.25358) 650 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:13.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:13 smithi138 bash[67894]: cluster 2024-07-28T19:54:12.200405+0000 mgr.y (mgr.25358) 650 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:15.050 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:54:15.474 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:54:15 smithi138 bash[59346]: logger=cleanup t=2024-07-28T19:54:15.170106485Z level=info msg="Completed cleanup jobs" duration=64.836778ms 2024-07-28T19:54:15.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:15 smithi100 bash[80199]: cluster 2024-07-28T19:54:14.201331+0000 mgr.y (mgr.25358) 651 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:15.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:15 smithi100 bash[80199]: audit 2024-07-28T19:54:15.303835+0000 mon.a (mon.0) 1607 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:15.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:15 smithi100 bash[80199]: audit 2024-07-28T19:54:15.308486+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:15.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:15 smithi100 bash[82644]: cluster 2024-07-28T19:54:14.201331+0000 mgr.y (mgr.25358) 651 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:15.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:15 smithi100 bash[82644]: audit 2024-07-28T19:54:15.303835+0000 mon.a (mon.0) 1607 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:15.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:15 smithi100 bash[82644]: audit 2024-07-28T19:54:15.308486+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:15.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:15 smithi138 bash[67894]: cluster 2024-07-28T19:54:14.201331+0000 mgr.y (mgr.25358) 651 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:15.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:15 smithi138 bash[67894]: audit 2024-07-28T19:54:15.303835+0000 mon.a (mon.0) 1607 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:15.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:15 smithi138 bash[67894]: audit 2024-07-28T19:54:15.308486+0000 mon.a (mon.0) 1608 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:16.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[80199]: audit 2024-07-28T19:54:15.565905+0000 mon.a (mon.0) 1609 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[80199]: audit 2024-07-28T19:54:15.570987+0000 mon.a (mon.0) 1610 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[80199]: audit 2024-07-28T19:54:15.958282+0000 mon.a (mon.0) 1611 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[80199]: audit 2024-07-28T19:54:15.959505+0000 mon.a (mon.0) 1612 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[80199]: audit 2024-07-28T19:54:15.966540+0000 mon.a (mon.0) 1613 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[80199]: cluster 2024-07-28T19:54:16.202302+0000 mgr.y (mgr.25358) 652 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[82644]: audit 2024-07-28T19:54:15.565905+0000 mon.a (mon.0) 1609 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[82644]: audit 2024-07-28T19:54:15.570987+0000 mon.a (mon.0) 1610 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[82644]: audit 2024-07-28T19:54:15.958282+0000 mon.a (mon.0) 1611 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[82644]: audit 2024-07-28T19:54:15.959505+0000 mon.a (mon.0) 1612 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:54:16.998 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[82644]: audit 2024-07-28T19:54:15.966540+0000 mon.a (mon.0) 1613 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:16.999 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:16 smithi100 bash[82644]: cluster 2024-07-28T19:54:16.202302+0000 mgr.y (mgr.25358) 652 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:16 smithi138 bash[67894]: audit 2024-07-28T19:54:15.565905+0000 mon.a (mon.0) 1609 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:16 smithi138 bash[67894]: audit 2024-07-28T19:54:15.570987+0000 mon.a (mon.0) 1610 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:16 smithi138 bash[67894]: audit 2024-07-28T19:54:15.958282+0000 mon.a (mon.0) 1611 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:54:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:16 smithi138 bash[67894]: audit 2024-07-28T19:54:15.959505+0000 mon.a (mon.0) 1612 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:54:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:16 smithi138 bash[67894]: audit 2024-07-28T19:54:15.966540+0000 mon.a (mon.0) 1613 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:54:17.057 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:16 smithi138 bash[67894]: cluster 2024-07-28T19:54:16.202302+0000 mgr.y (mgr.25358) 652 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:19.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:19 smithi138 bash[67894]: cluster 2024-07-28T19:54:18.203114+0000 mgr.y (mgr.25358) 653 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:19.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:19 smithi100 bash[80199]: cluster 2024-07-28T19:54:18.203114+0000 mgr.y (mgr.25358) 653 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:19.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:19 smithi100 bash[82644]: cluster 2024-07-28T19:54:18.203114+0000 mgr.y (mgr.25358) 653 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:19.834 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:54:21.437 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:21 smithi100 bash[80199]: cluster 2024-07-28T19:54:20.204088+0000 mgr.y (mgr.25358) 654 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:21.438 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:21 smithi100 bash[82644]: cluster 2024-07-28T19:54:20.204088+0000 mgr.y (mgr.25358) 654 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:21.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:21 smithi138 bash[67894]: cluster 2024-07-28T19:54:20.204088+0000 mgr.y (mgr.25358) 654 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:22.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:54:22 smithi138 bash[60615]: ts=2024-07-28T19:54:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:54:23.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:54:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:54:22] "GET /metrics HTTP/1.1" 200 34260 "" "Prometheus/2.43.0" 2024-07-28T19:54:23.297 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:54:23.298 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:54:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:23 smithi138 bash[67894]: cluster 2024-07-28T19:54:22.204879+0000 mgr.y (mgr.25358) 655 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:23.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:23 smithi138 bash[67894]: audit 2024-07-28T19:54:22.653553+0000 mon.a (mon.0) 1614 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:23.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:23 smithi100 bash[80199]: cluster 2024-07-28T19:54:22.204879+0000 mgr.y (mgr.25358) 655 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:23.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:23 smithi100 bash[80199]: audit 2024-07-28T19:54:22.653553+0000 mon.a (mon.0) 1614 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:23.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:23 smithi100 bash[82644]: cluster 2024-07-28T19:54:22.204879+0000 mgr.y (mgr.25358) 655 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:23.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:23 smithi100 bash[82644]: audit 2024-07-28T19:54:22.653553+0000 mon.a (mon.0) 1614 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:24.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:24 smithi100 bash[80199]: audit 2024-07-28T19:54:23.300635+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.100:0/852137066' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:24.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:24 smithi100 bash[82644]: audit 2024-07-28T19:54:23.300635+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.100:0/852137066' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:24.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:24 smithi138 bash[67894]: audit 2024-07-28T19:54:23.300635+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.100:0/852137066' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:25.558 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:25 smithi138 bash[67894]: cluster 2024-07-28T19:54:24.205834+0000 mgr.y (mgr.25358) 656 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:25.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:25 smithi100 bash[80199]: cluster 2024-07-28T19:54:24.205834+0000 mgr.y (mgr.25358) 656 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:25.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:25 smithi100 bash[82644]: cluster 2024-07-28T19:54:24.205834+0000 mgr.y (mgr.25358) 656 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:27.086 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:54:27.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:27 smithi100 bash[80199]: cluster 2024-07-28T19:54:26.206753+0000 mgr.y (mgr.25358) 657 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:27.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:27 smithi100 bash[82644]: cluster 2024-07-28T19:54:26.206753+0000 mgr.y (mgr.25358) 657 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:27.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:27 smithi138 bash[67894]: cluster 2024-07-28T19:54:26.206753+0000 mgr.y (mgr.25358) 657 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:29.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:29 smithi138 bash[67894]: cluster 2024-07-28T19:54:28.207454+0000 mgr.y (mgr.25358) 658 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:29.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:29 smithi100 bash[80199]: cluster 2024-07-28T19:54:28.207454+0000 mgr.y (mgr.25358) 658 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:29.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:29 smithi100 bash[82644]: cluster 2024-07-28T19:54:28.207454+0000 mgr.y (mgr.25358) 658 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:31 smithi138 bash[67894]: cluster 2024-07-28T19:54:30.208430+0000 mgr.y (mgr.25358) 659 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:31.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:31 smithi100 bash[80199]: cluster 2024-07-28T19:54:30.208430+0000 mgr.y (mgr.25358) 659 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:31.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:31 smithi100 bash[82644]: cluster 2024-07-28T19:54:30.208430+0000 mgr.y (mgr.25358) 659 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:31.879 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:54:32.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:54:32 smithi138 bash[60615]: ts=2024-07-28T19:54:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:54:33.134 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:54:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:54:32] "GET /metrics HTTP/1.1" 200 34260 "" "Prometheus/2.43.0" 2024-07-28T19:54:33.402 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:33 smithi100 bash[80199]: cluster 2024-07-28T19:54:32.209222+0000 mgr.y (mgr.25358) 660 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:33.402 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:33 smithi100 bash[82644]: cluster 2024-07-28T19:54:32.209222+0000 mgr.y (mgr.25358) 660 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:33 smithi138 bash[67894]: cluster 2024-07-28T19:54:32.209222+0000 mgr.y (mgr.25358) 660 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:35.281 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:54:35.281 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:54:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:35 smithi138 bash[67894]: cluster 2024-07-28T19:54:34.210175+0000 mgr.y (mgr.25358) 661 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:35 smithi138 bash[67894]: audit 2024-07-28T19:54:35.284588+0000 mon.a (mon.0) 1615 : audit [DBG] from='client.? 172.21.15.100:0/3331437535' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:35.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:35 smithi100 bash[80199]: cluster 2024-07-28T19:54:34.210175+0000 mgr.y (mgr.25358) 661 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:35.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:35 smithi100 bash[80199]: audit 2024-07-28T19:54:35.284588+0000 mon.a (mon.0) 1615 : audit [DBG] from='client.? 172.21.15.100:0/3331437535' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:35 smithi100 bash[82644]: cluster 2024-07-28T19:54:34.210175+0000 mgr.y (mgr.25358) 661 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:35.748 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:35 smithi100 bash[82644]: audit 2024-07-28T19:54:35.284588+0000 mon.a (mon.0) 1615 : audit [DBG] from='client.? 172.21.15.100:0/3331437535' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:37.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:37 smithi138 bash[67894]: cluster 2024-07-28T19:54:36.211063+0000 mgr.y (mgr.25358) 662 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:37.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:37 smithi100 bash[80199]: cluster 2024-07-28T19:54:36.211063+0000 mgr.y (mgr.25358) 662 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:37.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:37 smithi100 bash[82644]: cluster 2024-07-28T19:54:36.211063+0000 mgr.y (mgr.25358) 662 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:38.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:38 smithi100 bash[80199]: audit 2024-07-28T19:54:37.654039+0000 mon.a (mon.0) 1616 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:38.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:38 smithi100 bash[82644]: audit 2024-07-28T19:54:37.654039+0000 mon.a (mon.0) 1616 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:38 smithi138 bash[67894]: audit 2024-07-28T19:54:37.654039+0000 mon.a (mon.0) 1616 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:38.985 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:54:39.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:39 smithi100 bash[80199]: cluster 2024-07-28T19:54:38.211818+0000 mgr.y (mgr.25358) 663 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:39.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:39 smithi100 bash[82644]: cluster 2024-07-28T19:54:38.211818+0000 mgr.y (mgr.25358) 663 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:39 smithi138 bash[67894]: cluster 2024-07-28T19:54:38.211818+0000 mgr.y (mgr.25358) 663 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:41.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:41 smithi100 bash[80199]: cluster 2024-07-28T19:54:40.212793+0000 mgr.y (mgr.25358) 664 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:41.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:41 smithi100 bash[82644]: cluster 2024-07-28T19:54:40.212793+0000 mgr.y (mgr.25358) 664 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:41.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:41 smithi138 bash[67894]: cluster 2024-07-28T19:54:40.212793+0000 mgr.y (mgr.25358) 664 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:42.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:54:42 smithi138 bash[60615]: ts=2024-07-28T19:54:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:54:43.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:54:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:54:42] "GET /metrics HTTP/1.1" 200 34261 "" "Prometheus/2.43.0" 2024-07-28T19:54:43.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:43 smithi100 bash[80199]: cluster 2024-07-28T19:54:42.213413+0000 mgr.y (mgr.25358) 665 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:43.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:43 smithi100 bash[82644]: cluster 2024-07-28T19:54:42.213413+0000 mgr.y (mgr.25358) 665 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:43.767 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:54:43.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:43 smithi138 bash[67894]: cluster 2024-07-28T19:54:42.213413+0000 mgr.y (mgr.25358) 665 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:45.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:45 smithi100 bash[80199]: cluster 2024-07-28T19:54:44.214352+0000 mgr.y (mgr.25358) 666 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:45.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:45 smithi100 bash[82644]: cluster 2024-07-28T19:54:44.214352+0000 mgr.y (mgr.25358) 666 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:45.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:45 smithi138 bash[67894]: cluster 2024-07-28T19:54:44.214352+0000 mgr.y (mgr.25358) 666 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:47.349 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:54:47.349 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:54:47.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:47 smithi100 bash[80199]: cluster 2024-07-28T19:54:46.215255+0000 mgr.y (mgr.25358) 667 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:47.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:47 smithi100 bash[82644]: cluster 2024-07-28T19:54:46.215255+0000 mgr.y (mgr.25358) 667 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:47 smithi138 bash[67894]: cluster 2024-07-28T19:54:46.215255+0000 mgr.y (mgr.25358) 667 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:48.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:48 smithi100 bash[80199]: audit 2024-07-28T19:54:47.352164+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.100:0/2222855151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:48.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:48 smithi100 bash[82644]: audit 2024-07-28T19:54:47.352164+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.100:0/2222855151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:48.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:48 smithi138 bash[67894]: audit 2024-07-28T19:54:47.352164+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.100:0/2222855151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:54:49.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:49 smithi100 bash[80199]: cluster 2024-07-28T19:54:48.216140+0000 mgr.y (mgr.25358) 668 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:49.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:49 smithi100 bash[82644]: cluster 2024-07-28T19:54:48.216140+0000 mgr.y (mgr.25358) 668 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:49.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:49 smithi138 bash[67894]: cluster 2024-07-28T19:54:48.216140+0000 mgr.y (mgr.25358) 668 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:51.069 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:54:51.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:51 smithi100 bash[80199]: cluster 2024-07-28T19:54:50.217327+0000 mgr.y (mgr.25358) 669 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:51 smithi100 bash[82644]: cluster 2024-07-28T19:54:50.217327+0000 mgr.y (mgr.25358) 669 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:51 smithi138 bash[67894]: cluster 2024-07-28T19:54:50.217327+0000 mgr.y (mgr.25358) 669 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:52.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:54:52 smithi138 bash[60615]: ts=2024-07-28T19:54:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:54:53.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:54:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:54:52] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-07-28T19:54:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:53 smithi100 bash[80199]: cluster 2024-07-28T19:54:52.218090+0000 mgr.y (mgr.25358) 670 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:53 smithi100 bash[80199]: audit 2024-07-28T19:54:52.654562+0000 mon.a (mon.0) 1617 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:53.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:53 smithi100 bash[82644]: cluster 2024-07-28T19:54:52.218090+0000 mgr.y (mgr.25358) 670 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:53.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:53 smithi100 bash[82644]: audit 2024-07-28T19:54:52.654562+0000 mon.a (mon.0) 1617 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:53 smithi138 bash[67894]: cluster 2024-07-28T19:54:52.218090+0000 mgr.y (mgr.25358) 670 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:53 smithi138 bash[67894]: audit 2024-07-28T19:54:52.654562+0000 mon.a (mon.0) 1617 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:54:55.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:55 smithi100 bash[80199]: cluster 2024-07-28T19:54:54.219030+0000 mgr.y (mgr.25358) 671 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:55.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:55 smithi100 bash[82644]: cluster 2024-07-28T19:54:54.219030+0000 mgr.y (mgr.25358) 671 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:55 smithi138 bash[67894]: cluster 2024-07-28T19:54:54.219030+0000 mgr.y (mgr.25358) 671 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:55.851 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:54:57.760 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:57 smithi100 bash[80199]: cluster 2024-07-28T19:54:56.219856+0000 mgr.y (mgr.25358) 672 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:57.760 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:57 smithi100 bash[82644]: cluster 2024-07-28T19:54:56.219856+0000 mgr.y (mgr.25358) 672 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:57.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:57 smithi138 bash[67894]: cluster 2024-07-28T19:54:56.219856+0000 mgr.y (mgr.25358) 672 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:59.446 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:54:59.446 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:54:59.647 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:54:59 smithi100 bash[80199]: cluster 2024-07-28T19:54:58.220615+0000 mgr.y (mgr.25358) 673 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:59.647 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:54:59 smithi100 bash[82644]: cluster 2024-07-28T19:54:58.220615+0000 mgr.y (mgr.25358) 673 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:54:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:54:59 smithi138 bash[67894]: cluster 2024-07-28T19:54:58.220615+0000 mgr.y (mgr.25358) 673 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:00.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:00 smithi100 bash[80199]: audit 2024-07-28T19:54:59.449590+0000 mon.a (mon.0) 1618 : audit [DBG] from='client.? 172.21.15.100:0/2843369077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:00.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:00 smithi100 bash[82644]: audit 2024-07-28T19:54:59.449590+0000 mon.a (mon.0) 1618 : audit [DBG] from='client.? 172.21.15.100:0/2843369077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:00.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:00 smithi138 bash[67894]: audit 2024-07-28T19:54:59.449590+0000 mon.a (mon.0) 1618 : audit [DBG] from='client.? 172.21.15.100:0/2843369077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:01.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:01 smithi100 bash[80199]: cluster 2024-07-28T19:55:00.221717+0000 mgr.y (mgr.25358) 674 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:01.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:01 smithi100 bash[82644]: cluster 2024-07-28T19:55:00.221717+0000 mgr.y (mgr.25358) 674 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:01.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:01 smithi138 bash[67894]: cluster 2024-07-28T19:55:00.221717+0000 mgr.y (mgr.25358) 674 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:55:02 smithi138 bash[60615]: ts=2024-07-28T19:55:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:55:03.213 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:55:03.224 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:55:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:55:02] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-07-28T19:55:03.497 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:03 smithi100 bash[80199]: cluster 2024-07-28T19:55:02.222346+0000 mgr.y (mgr.25358) 675 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:03.497 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:03 smithi100 bash[82644]: cluster 2024-07-28T19:55:02.222346+0000 mgr.y (mgr.25358) 675 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:03.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:03 smithi138 bash[67894]: cluster 2024-07-28T19:55:02.222346+0000 mgr.y (mgr.25358) 675 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:05.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:05 smithi100 bash[80199]: cluster 2024-07-28T19:55:04.223229+0000 mgr.y (mgr.25358) 676 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:05.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:05 smithi100 bash[82644]: cluster 2024-07-28T19:55:04.223229+0000 mgr.y (mgr.25358) 676 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:05.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:05 smithi138 bash[67894]: cluster 2024-07-28T19:55:04.223229+0000 mgr.y (mgr.25358) 676 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:07.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:07 smithi100 bash[80199]: cluster 2024-07-28T19:55:06.224140+0000 mgr.y (mgr.25358) 677 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:07.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:07 smithi100 bash[82644]: cluster 2024-07-28T19:55:06.224140+0000 mgr.y (mgr.25358) 677 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:07.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:07 smithi138 bash[67894]: cluster 2024-07-28T19:55:06.224140+0000 mgr.y (mgr.25358) 677 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:08.001 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:55:08.749 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:08 smithi100 bash[80199]: audit 2024-07-28T19:55:07.654847+0000 mon.a (mon.0) 1619 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:08.750 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:08 smithi100 bash[82644]: audit 2024-07-28T19:55:07.654847+0000 mon.a (mon.0) 1619 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:08.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:08 smithi138 bash[67894]: audit 2024-07-28T19:55:07.654847+0000 mon.a (mon.0) 1619 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:09.612 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:09 smithi100 bash[80199]: cluster 2024-07-28T19:55:08.224977+0000 mgr.y (mgr.25358) 678 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:09.612 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:09 smithi100 bash[82644]: cluster 2024-07-28T19:55:08.224977+0000 mgr.y (mgr.25358) 678 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:09.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:09 smithi138 bash[67894]: cluster 2024-07-28T19:55:08.224977+0000 mgr.y (mgr.25358) 678 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:11.625 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:55:11.625 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:55:11.636 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:11 smithi100 bash[80199]: cluster 2024-07-28T19:55:10.225903+0000 mgr.y (mgr.25358) 679 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:11.637 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:11 smithi100 bash[82644]: cluster 2024-07-28T19:55:10.225903+0000 mgr.y (mgr.25358) 679 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:11.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:11 smithi138 bash[67894]: cluster 2024-07-28T19:55:10.225903+0000 mgr.y (mgr.25358) 679 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:12.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:12 smithi100 bash[80199]: audit 2024-07-28T19:55:11.628626+0000 mon.a (mon.0) 1620 : audit [DBG] from='client.? 172.21.15.100:0/3465367973' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:12.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:12 smithi100 bash[82644]: audit 2024-07-28T19:55:11.628626+0000 mon.a (mon.0) 1620 : audit [DBG] from='client.? 172.21.15.100:0/3465367973' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:12.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:12 smithi138 bash[67894]: audit 2024-07-28T19:55:11.628626+0000 mon.a (mon.0) 1620 : audit [DBG] from='client.? 172.21.15.100:0/3465367973' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:12.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:55:12 smithi138 bash[60615]: ts=2024-07-28T19:55:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:55:13.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:55:12 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:55:12] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-07-28T19:55:13.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:13 smithi100 bash[82644]: cluster 2024-07-28T19:55:12.226485+0000 mgr.y (mgr.25358) 680 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:13.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:13 smithi100 bash[80199]: cluster 2024-07-28T19:55:12.226485+0000 mgr.y (mgr.25358) 680 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:13.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:13 smithi138 bash[67894]: cluster 2024-07-28T19:55:12.226485+0000 mgr.y (mgr.25358) 680 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:15.320 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:55:15.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:15 smithi100 bash[80199]: cluster 2024-07-28T19:55:14.227447+0000 mgr.y (mgr.25358) 681 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:15.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:15 smithi100 bash[82644]: cluster 2024-07-28T19:55:14.227447+0000 mgr.y (mgr.25358) 681 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:15.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:15 smithi138 bash[67894]: cluster 2024-07-28T19:55:14.227447+0000 mgr.y (mgr.25358) 681 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:16.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:16 smithi100 bash[80199]: audit 2024-07-28T19:55:16.090850+0000 mon.a (mon.0) 1621 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:55:16.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:16 smithi100 bash[82644]: audit 2024-07-28T19:55:16.090850+0000 mon.a (mon.0) 1621 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:55:16.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:16 smithi138 bash[67894]: audit 2024-07-28T19:55:16.090850+0000 mon.a (mon.0) 1621 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-07-28T19:55:17.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:17 smithi100 bash[80199]: cluster 2024-07-28T19:55:16.228202+0000 mgr.y (mgr.25358) 682 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:17.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:17 smithi100 bash[82644]: cluster 2024-07-28T19:55:16.228202+0000 mgr.y (mgr.25358) 682 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:17.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:17 smithi138 bash[67894]: cluster 2024-07-28T19:55:16.228202+0000 mgr.y (mgr.25358) 682 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:19.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:19 smithi100 bash[80199]: cluster 2024-07-28T19:55:18.228952+0000 mgr.y (mgr.25358) 683 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:19.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:19 smithi100 bash[82644]: cluster 2024-07-28T19:55:18.228952+0000 mgr.y (mgr.25358) 683 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:19.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:19 smithi138 bash[67894]: cluster 2024-07-28T19:55:18.228952+0000 mgr.y (mgr.25358) 683 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:20.108 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:55:21.652 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:21 smithi100 bash[82644]: cluster 2024-07-28T19:55:20.229560+0000 mgr.y (mgr.25358) 684 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:21.652 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:21 smithi100 bash[82644]: audit 2024-07-28T19:55:20.776408+0000 mon.a (mon.0) 1622 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:21.652 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:21 smithi100 bash[82644]: audit 2024-07-28T19:55:20.785422+0000 mon.a (mon.0) 1623 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:21.652 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:21 smithi100 bash[80199]: cluster 2024-07-28T19:55:20.229560+0000 mgr.y (mgr.25358) 684 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:21.652 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:21 smithi100 bash[80199]: audit 2024-07-28T19:55:20.776408+0000 mon.a (mon.0) 1622 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:21.652 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:21 smithi100 bash[80199]: audit 2024-07-28T19:55:20.785422+0000 mon.a (mon.0) 1623 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:21 smithi138 bash[67894]: cluster 2024-07-28T19:55:20.229560+0000 mgr.y (mgr.25358) 684 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:21.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:21 smithi138 bash[67894]: audit 2024-07-28T19:55:20.776408+0000 mon.a (mon.0) 1622 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:21.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:21 smithi138 bash[67894]: audit 2024-07-28T19:55:20.785422+0000 mon.a (mon.0) 1623 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:22.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:55:22 smithi138 bash[60615]: ts=2024-07-28T19:55:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:55:23.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:55:22] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-07-28T19:55:23.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[80199]: audit 2024-07-28T19:55:21.964446+0000 mon.a (mon.0) 1624 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[80199]: audit 2024-07-28T19:55:21.973745+0000 mon.a (mon.0) 1625 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[80199]: cluster 2024-07-28T19:55:22.229940+0000 mgr.y (mgr.25358) 685 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:23.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[80199]: audit 2024-07-28T19:55:22.421872+0000 mon.a (mon.0) 1626 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:55:23.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[80199]: audit 2024-07-28T19:55:22.423653+0000 mon.a (mon.0) 1627 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[80199]: audit 2024-07-28T19:55:22.430519+0000 mon.a (mon.0) 1628 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[80199]: audit 2024-07-28T19:55:22.655275+0000 mon.a (mon.0) 1629 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[82644]: audit 2024-07-28T19:55:21.964446+0000 mon.a (mon.0) 1624 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[82644]: audit 2024-07-28T19:55:21.973745+0000 mon.a (mon.0) 1625 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[82644]: cluster 2024-07-28T19:55:22.229940+0000 mgr.y (mgr.25358) 685 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[82644]: audit 2024-07-28T19:55:22.421872+0000 mon.a (mon.0) 1626 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[82644]: audit 2024-07-28T19:55:22.423653+0000 mon.a (mon.0) 1627 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[82644]: audit 2024-07-28T19:55:22.430519+0000 mon.a (mon.0) 1628 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.248 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:22 smithi100 bash[82644]: audit 2024-07-28T19:55:22.655275+0000 mon.a (mon.0) 1629 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:23.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:22 smithi138 bash[67894]: audit 2024-07-28T19:55:21.964446+0000 mon.a (mon.0) 1624 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:22 smithi138 bash[67894]: audit 2024-07-28T19:55:21.973745+0000 mon.a (mon.0) 1625 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:22 smithi138 bash[67894]: cluster 2024-07-28T19:55:22.229940+0000 mgr.y (mgr.25358) 685 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:23.307 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:22 smithi138 bash[67894]: audit 2024-07-28T19:55:22.421872+0000 mon.a (mon.0) 1626 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-07-28T19:55:23.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:22 smithi138 bash[67894]: audit 2024-07-28T19:55:22.423653+0000 mon.a (mon.0) 1627 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-07-28T19:55:23.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:22 smithi138 bash[67894]: audit 2024-07-28T19:55:22.430519+0000 mon.a (mon.0) 1628 : audit [INF] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' 2024-07-28T19:55:23.308 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:22 smithi138 bash[67894]: audit 2024-07-28T19:55:22.655275+0000 mon.a (mon.0) 1629 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:23.729 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:55:23.729 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:55:23.997 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:23 smithi100 bash[82644]: audit 2024-07-28T19:55:23.732800+0000 mon.a (mon.0) 1630 : audit [DBG] from='client.? 172.21.15.100:0/241153241' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:23.997 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:23 smithi100 bash[80199]: audit 2024-07-28T19:55:23.732800+0000 mon.a (mon.0) 1630 : audit [DBG] from='client.? 172.21.15.100:0/241153241' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:24.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:23 smithi138 bash[67894]: audit 2024-07-28T19:55:23.732800+0000 mon.a (mon.0) 1630 : audit [DBG] from='client.? 172.21.15.100:0/241153241' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:25.247 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:24 smithi100 bash[80199]: cluster 2024-07-28T19:55:24.230852+0000 mgr.y (mgr.25358) 686 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:25.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:24 smithi100 bash[82644]: cluster 2024-07-28T19:55:24.230852+0000 mgr.y (mgr.25358) 686 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:25.306 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:24 smithi138 bash[67894]: cluster 2024-07-28T19:55:24.230852+0000 mgr.y (mgr.25358) 686 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:27.513 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:55:27.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:27 smithi138 bash[67894]: cluster 2024-07-28T19:55:26.231690+0000 mgr.y (mgr.25358) 687 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:27.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:27 smithi100 bash[80199]: cluster 2024-07-28T19:55:26.231690+0000 mgr.y (mgr.25358) 687 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:27.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:27 smithi100 bash[82644]: cluster 2024-07-28T19:55:26.231690+0000 mgr.y (mgr.25358) 687 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:29.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:29 smithi138 bash[67894]: cluster 2024-07-28T19:55:28.232431+0000 mgr.y (mgr.25358) 688 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:29.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:29 smithi100 bash[80199]: cluster 2024-07-28T19:55:28.232431+0000 mgr.y (mgr.25358) 688 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:29.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:29 smithi100 bash[82644]: cluster 2024-07-28T19:55:28.232431+0000 mgr.y (mgr.25358) 688 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:31.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:31 smithi138 bash[67894]: cluster 2024-07-28T19:55:30.233561+0000 mgr.y (mgr.25358) 689 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:31.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:31 smithi100 bash[80199]: cluster 2024-07-28T19:55:30.233561+0000 mgr.y (mgr.25358) 689 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:31.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:31 smithi100 bash[82644]: cluster 2024-07-28T19:55:30.233561+0000 mgr.y (mgr.25358) 689 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:32.302 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:55:32.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:55:32 smithi138 bash[60615]: ts=2024-07-28T19:55:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:55:33.247 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:55:32 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:55:32] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-07-28T19:55:33.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:33 smithi138 bash[67894]: cluster 2024-07-28T19:55:32.234221+0000 mgr.y (mgr.25358) 690 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:33.625 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:33 smithi100 bash[80199]: cluster 2024-07-28T19:55:32.234221+0000 mgr.y (mgr.25358) 690 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:33.625 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:33 smithi100 bash[82644]: cluster 2024-07-28T19:55:32.234221+0000 mgr.y (mgr.25358) 690 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:35.555 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:35 smithi100 bash[80199]: cluster 2024-07-28T19:55:34.235155+0000 mgr.y (mgr.25358) 691 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:35.555 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:35 smithi100 bash[82644]: cluster 2024-07-28T19:55:34.235155+0000 mgr.y (mgr.25358) 691 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:35.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:35 smithi138 bash[67894]: cluster 2024-07-28T19:55:34.235155+0000 mgr.y (mgr.25358) 691 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:35.889 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:55:35.890 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:55:36.633 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:36 smithi100 bash[80199]: audit 2024-07-28T19:55:35.893190+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.100:0/678011924' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:36.633 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:36 smithi100 bash[82644]: audit 2024-07-28T19:55:35.893190+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.100:0/678011924' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:36.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:36 smithi138 bash[67894]: audit 2024-07-28T19:55:35.893190+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.100:0/678011924' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:37.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:37 smithi100 bash[80199]: cluster 2024-07-28T19:55:36.235965+0000 mgr.y (mgr.25358) 692 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:37.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:37 smithi100 bash[82644]: cluster 2024-07-28T19:55:36.235965+0000 mgr.y (mgr.25358) 692 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:37.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:37 smithi138 bash[67894]: cluster 2024-07-28T19:55:36.235965+0000 mgr.y (mgr.25358) 692 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:38.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:38 smithi100 bash[80199]: audit 2024-07-28T19:55:37.655633+0000 mon.a (mon.0) 1631 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:38.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:38 smithi100 bash[82644]: audit 2024-07-28T19:55:37.655633+0000 mon.a (mon.0) 1631 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:38.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:38 smithi138 bash[67894]: audit 2024-07-28T19:55:37.655633+0000 mon.a (mon.0) 1631 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:39.635 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:55:39.651 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:39 smithi100 bash[80199]: cluster 2024-07-28T19:55:38.236677+0000 mgr.y (mgr.25358) 693 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:39.651 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:39 smithi100 bash[82644]: cluster 2024-07-28T19:55:38.236677+0000 mgr.y (mgr.25358) 693 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:39.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:39 smithi138 bash[67894]: cluster 2024-07-28T19:55:38.236677+0000 mgr.y (mgr.25358) 693 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:41.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:41 smithi100 bash[80199]: cluster 2024-07-28T19:55:40.237739+0000 mgr.y (mgr.25358) 694 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:41.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:41 smithi100 bash[82644]: cluster 2024-07-28T19:55:40.237739+0000 mgr.y (mgr.25358) 694 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:41.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:41 smithi138 bash[67894]: cluster 2024-07-28T19:55:40.237739+0000 mgr.y (mgr.25358) 694 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:42.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:55:42 smithi138 bash[60615]: ts=2024-07-28T19:55:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:55:43.246 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:55:42 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:55:42] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-07-28T19:55:43.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:43 smithi100 bash[80199]: cluster 2024-07-28T19:55:42.238365+0000 mgr.y (mgr.25358) 695 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:43.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:43 smithi100 bash[82644]: cluster 2024-07-28T19:55:42.238365+0000 mgr.y (mgr.25358) 695 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:43.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:43 smithi138 bash[67894]: cluster 2024-07-28T19:55:42.238365+0000 mgr.y (mgr.25358) 695 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:44.418 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:55:45.598 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:45 smithi100 bash[80199]: cluster 2024-07-28T19:55:44.239314+0000 mgr.y (mgr.25358) 696 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:45.598 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:45 smithi100 bash[82644]: cluster 2024-07-28T19:55:44.239314+0000 mgr.y (mgr.25358) 696 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:45.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:45 smithi138 bash[67894]: cluster 2024-07-28T19:55:44.239314+0000 mgr.y (mgr.25358) 696 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:47.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:47 smithi100 bash[80199]: cluster 2024-07-28T19:55:46.240170+0000 mgr.y (mgr.25358) 697 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:47.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:47 smithi100 bash[82644]: cluster 2024-07-28T19:55:46.240170+0000 mgr.y (mgr.25358) 697 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:47.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:47 smithi138 bash[67894]: cluster 2024-07-28T19:55:46.240170+0000 mgr.y (mgr.25358) 697 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:47.896 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:55:47.896 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:55:48.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:48 smithi100 bash[80199]: audit 2024-07-28T19:55:47.899604+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.100:0/3508519687' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:48.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:48 smithi100 bash[82644]: audit 2024-07-28T19:55:47.899604+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.100:0/3508519687' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:48.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:48 smithi138 bash[67894]: audit 2024-07-28T19:55:47.899604+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.100:0/3508519687' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:55:49.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:49 smithi100 bash[80199]: cluster 2024-07-28T19:55:48.240916+0000 mgr.y (mgr.25358) 698 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:49.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:49 smithi100 bash[82644]: cluster 2024-07-28T19:55:48.240916+0000 mgr.y (mgr.25358) 698 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:49.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:49 smithi138 bash[67894]: cluster 2024-07-28T19:55:48.240916+0000 mgr.y (mgr.25358) 698 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:51.590 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 -- ceph health --format=json 2024-07-28T19:55:51.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:51 smithi100 bash[80199]: cluster 2024-07-28T19:55:50.242024+0000 mgr.y (mgr.25358) 699 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:51.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:51 smithi100 bash[82644]: cluster 2024-07-28T19:55:50.242024+0000 mgr.y (mgr.25358) 699 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:51.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:51 smithi138 bash[67894]: cluster 2024-07-28T19:55:50.242024+0000 mgr.y (mgr.25358) 699 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:52.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:55:52 smithi138 bash[60615]: ts=2024-07-28T19:55:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:55:53.246 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:55:52 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:55:52] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-07-28T19:55:53.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:53 smithi100 bash[80199]: cluster 2024-07-28T19:55:52.242791+0000 mgr.y (mgr.25358) 700 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:53.747 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:53 smithi100 bash[80199]: audit 2024-07-28T19:55:52.656347+0000 mon.a (mon.0) 1632 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:53.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:53 smithi100 bash[82644]: cluster 2024-07-28T19:55:52.242791+0000 mgr.y (mgr.25358) 700 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:53.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:53 smithi100 bash[82644]: audit 2024-07-28T19:55:52.656347+0000 mon.a (mon.0) 1632 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:53.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:53 smithi138 bash[67894]: cluster 2024-07-28T19:55:52.242791+0000 mgr.y (mgr.25358) 700 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:53.807 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:53 smithi138 bash[67894]: audit 2024-07-28T19:55:52.656347+0000 mon.a (mon.0) 1632 : audit [DBG] from='mgr.25358 172.21.15.100:0/3788345769' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-07-28T19:55:55.746 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:55 smithi100 bash[80199]: cluster 2024-07-28T19:55:54.243780+0000 mgr.y (mgr.25358) 701 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:55.747 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:55 smithi100 bash[82644]: cluster 2024-07-28T19:55:54.243780+0000 mgr.y (mgr.25358) 701 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:55.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:55 smithi138 bash[67894]: cluster 2024-07-28T19:55:54.243780+0000 mgr.y (mgr.25358) 701 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:56.374 INFO:teuthology.orchestra.run.smithi100.stderr:Inferring config /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/mon.a/config 2024-07-28T19:55:57.646 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:57 smithi100 bash[80199]: cluster 2024-07-28T19:55:56.244631+0000 mgr.y (mgr.25358) 702 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:57.647 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:57 smithi100 bash[82644]: cluster 2024-07-28T19:55:56.244631+0000 mgr.y (mgr.25358) 702 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:57.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:57 smithi138 bash[67894]: cluster 2024-07-28T19:55:56.244631+0000 mgr.y (mgr.25358) 702 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:59.636 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-07-28T19:55:59.636 INFO:teuthology.orchestra.run.smithi100.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-07-28T19:55:59.647 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:55:59 smithi100 bash[80199]: cluster 2024-07-28T19:55:58.245176+0000 mgr.y (mgr.25358) 703 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:59.647 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:55:59 smithi100 bash[82644]: cluster 2024-07-28T19:55:58.245176+0000 mgr.y (mgr.25358) 703 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:55:59.806 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:55:59 smithi138 bash[67894]: cluster 2024-07-28T19:55:58.245176+0000 mgr.y (mgr.25358) 703 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-07-28T19:56:00.371 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_a8aed60fc62d4ff39d2216b360f018eba7518cfe/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_a8aed60fc62d4ff39d2216b360f018eba7518cfe/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_a3bbd7289877bdcce87fd1f79da1a2d6578dde36/qa/tasks/ceph.py", line 1494, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_a3bbd7289877bdcce87fd1f79da1a2d6578dde36/qa/tasks/ceph_manager.py", line 3191, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-07-28T19:56:00.593 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=50a1708004f549b4a1067057efcd10d0 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_a8aed60fc62d4ff39d2216b360f018eba7518cfe/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_a8aed60fc62d4ff39d2216b360f018eba7518cfe/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_a3bbd7289877bdcce87fd1f79da1a2d6578dde36/qa/tasks/ceph.py", line 1494, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_a3bbd7289877bdcce87fd1f79da1a2d6578dde36/qa/tasks/ceph_manager.py", line 3191, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-07-28T19:56:00.595 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-07-28T19:56:00.606 INFO:tasks.cephadm:Teardown begin 2024-07-28T19:56:00.606 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-07-28T19:56:00.620 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:56:00 smithi100 bash[80199]: audit 2024-07-28T19:55:59.639946+0000 mon.a (mon.0) 1633 : audit [DBG] from='client.? 172.21.15.100:0/2049877052' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:56:00.620 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:56:00 smithi100 bash[82644]: audit 2024-07-28T19:55:59.639946+0000 mon.a (mon.0) 1633 : audit [DBG] from='client.? 172.21.15.100:0/2049877052' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:56:00.623 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-07-28T19:56:00.634 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:56:00 smithi138 bash[67894]: audit 2024-07-28T19:55:59.639946+0000 mon.a (mon.0) 1633 : audit [DBG] from='client.? 172.21.15.100:0/2049877052' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-07-28T19:56:00.638 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-07-28T19:56:00.638 DEBUG:teuthology.orchestra.run.smithi100:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-07-28T19:56:00.669 DEBUG:teuthology.orchestra.run.smithi138:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-07-28T19:56:00.685 INFO:tasks.cephadm:Stopping all daemons... 2024-07-28T19:56:00.686 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-07-28T19:56:00.686 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a 2024-07-28T19:56:00.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:56:00 smithi100 systemd[1]: Stopping Ceph mon.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:00.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:56:00 smithi100 bash[80199]: debug 2024-07-28T19:56:00.863+0000 7f8fc9d12640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-28T19:56:00.882 INFO:journalctl@ceph.mon.a.smithi100.stdout:Jul 28 19:56:00 smithi100 bash[80199]: debug 2024-07-28T19:56:00.863+0000 7f8fc9d12640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-07-28T19:56:00.996 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:00 smithi100 bash[73305]: [28/Jul/2024:19:56:00] ENGINE Bus STOPPING 2024-07-28T19:56:01.651 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:01 smithi100 bash[73305]: [28/Jul/2024:19:56:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:56:01.651 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:01 smithi100 bash[73305]: [28/Jul/2024:19:56:01] ENGINE Bus STOPPED 2024-07-28T19:56:01.651 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:01 smithi100 bash[73305]: [28/Jul/2024:19:56:01] ENGINE Bus STARTING 2024-07-28T19:56:01.651 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:01 smithi100 bash[73305]: [28/Jul/2024:19:56:01] ENGINE Serving on http://:::9283 2024-07-28T19:56:01.651 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:01 smithi100 bash[73305]: [28/Jul/2024:19:56:01] ENGINE Bus STARTED 2024-07-28T19:56:01.885 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.a.service' 2024-07-28T19:56:01.916 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:01.916 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-07-28T19:56:01.916 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-07-28T19:56:01.917 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.c 2024-07-28T19:56:02.246 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[73305]: [28/Jul/2024:19:56:02] ENGINE Bus STOPPING 2024-07-28T19:56:02.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:56:01 smithi100 systemd[1]: Stopping Ceph mon.c for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:02.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[82644]: debug 2024-07-28T19:56:02.103+0000 7efd43b8d640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-28T19:56:02.247 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[82644]: debug 2024-07-28T19:56:02.103+0000 7efd43b8d640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-07-28T19:56:02.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[73305]: [28/Jul/2024:19:56:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-07-28T19:56:02.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[73305]: [28/Jul/2024:19:56:02] ENGINE Bus STOPPED 2024-07-28T19:56:02.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[73305]: [28/Jul/2024:19:56:02] ENGINE Bus STARTING 2024-07-28T19:56:02.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[73305]: [28/Jul/2024:19:56:02] ENGINE Serving on http://:::9283 2024-07-28T19:56:02.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[73305]: [28/Jul/2024:19:56:02] ENGINE Bus STARTED 2024-07-28T19:56:02.806 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:56:02 smithi138 bash[60615]: ts=2024-07-28T19:56:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:56:03.246 INFO:journalctl@ceph.mon.c.smithi100.stdout:Jul 28 19:56:03 smithi100 bash[123665]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-mon-c 2024-07-28T19:56:03.246 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:02 smithi100 bash[73305]: ::ffff:172.21.15.138 - - [28/Jul/2024:19:56:02] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-07-28T19:56:03.262 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.c.service' 2024-07-28T19:56:03.287 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:03.287 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-07-28T19:56:03.288 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-07-28T19:56:03.288 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.b 2024-07-28T19:56:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:56:03 smithi138 systemd[1]: Stopping Ceph mon.b for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:03.556 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:56:03 smithi138 bash[67894]: debug 2024-07-28T19:56:03.504+0000 7f9dfdcea640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-07-28T19:56:03.557 INFO:journalctl@ceph.mon.b.smithi138.stdout:Jul 28 19:56:03 smithi138 bash[67894]: debug 2024-07-28T19:56:03.504+0000 7f9dfdcea640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-07-28T19:56:04.441 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mon.b.service' 2024-07-28T19:56:04.475 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:04.475 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-07-28T19:56:04.475 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-07-28T19:56:04.475 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y 2024-07-28T19:56:04.746 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:04 smithi100 systemd[1]: Stopping Ceph mgr.y for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:04.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:04 smithi100 bash[73305]: debug 2024-07-28T19:56:04.623+0000 7f96d36b7640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:56:04.747 INFO:journalctl@ceph.mgr.y.smithi100.stdout:Jul 28 19:56:04 smithi100 bash[73305]: debug 2024-07-28T19:56:04.623+0000 7f96d36b7640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-07-28T19:56:05.491 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.y.service' 2024-07-28T19:56:05.516 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:05.516 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-07-28T19:56:05.517 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-07-28T19:56:05.517 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.x 2024-07-28T19:56:05.806 INFO:journalctl@ceph.mgr.x.smithi138.stdout:Jul 28 19:56:05 smithi138 systemd[1]: Stopping Ceph mgr.x for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:06.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@mgr.x.service' 2024-07-28T19:56:06.613 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:06.614 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-07-28T19:56:06.614 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-07-28T19:56:06.614 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.0 2024-07-28T19:56:06.996 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:56:06 smithi100 systemd[1]: Stopping Ceph osd.0 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:06.997 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:56:06 smithi100 bash[89627]: debug 2024-07-28T19:56:06.759+0000 7f936f28f640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:56:06.997 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:56:06 smithi100 bash[89627]: debug 2024-07-28T19:56:06.759+0000 7f936f28f640 -1 osd.0 961 *** Got signal Terminated *** 2024-07-28T19:56:06.997 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:56:06 smithi100 bash[89627]: debug 2024-07-28T19:56:06.759+0000 7f936f28f640 -1 osd.0 961 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:56:12.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:56:12 smithi138 bash[60615]: ts=2024-07-28T19:56:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9170a172-4d15-11ef-bcc8-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-07-28T19:56:13.246 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:56:12 smithi100 bash[123876]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-0 2024-07-28T19:56:13.247 INFO:journalctl@ceph.osd.0.smithi100.stdout:Jul 28 19:56:13 smithi100 bash[123939]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-0 2024-07-28T19:56:15.373 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.0.service' 2024-07-28T19:56:15.400 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:15.400 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-07-28T19:56:15.401 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-07-28T19:56:15.401 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.1 2024-07-28T19:56:15.747 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:56:15 smithi100 systemd[1]: Stopping Ceph osd.1 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:15.747 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:56:15 smithi100 bash[94549]: debug 2024-07-28T19:56:15.594+0000 7efe1d2c9640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:56:15.747 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:56:15 smithi100 bash[94549]: debug 2024-07-28T19:56:15.594+0000 7efe1d2c9640 -1 osd.1 961 *** Got signal Terminated *** 2024-07-28T19:56:15.747 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:56:15 smithi100 bash[94549]: debug 2024-07-28T19:56:15.594+0000 7efe1d2c9640 -1 osd.1 961 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:56:20.496 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:20 smithi100 bash[72716]: ts=2024-07-28T19:56:20.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:56:20.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:20 smithi100 bash[72716]: ts=2024-07-28T19:56:20.208Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:56:21.926 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:56:21 smithi100 bash[124070]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-1 2024-07-28T19:56:21.926 INFO:journalctl@ceph.osd.1.smithi100.stdout:Jul 28 19:56:21 smithi100 bash[124135]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-1 2024-07-28T19:56:23.874 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.1.service' 2024-07-28T19:56:23.902 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:23.902 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-07-28T19:56:23.902 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-07-28T19:56:23.903 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.2 2024-07-28T19:56:24.246 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:56:23 smithi100 systemd[1]: Stopping Ceph osd.2 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:24.246 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:56:24 smithi100 bash[99897]: debug 2024-07-28T19:56:24.066+0000 7fc2fd47f640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:56:24.247 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:56:24 smithi100 bash[99897]: debug 2024-07-28T19:56:24.066+0000 7fc2fd47f640 -1 osd.2 961 *** Got signal Terminated *** 2024-07-28T19:56:24.247 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:56:24 smithi100 bash[99897]: debug 2024-07-28T19:56:24.066+0000 7fc2fd47f640 -1 osd.2 961 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:56:30.496 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:56:30 smithi100 bash[124262]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-2 2024-07-28T19:56:30.497 INFO:journalctl@ceph.osd.2.smithi100.stdout:Jul 28 19:56:30 smithi100 bash[124325]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-2 2024-07-28T19:56:30.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:30 smithi100 bash[72716]: ts=2024-07-28T19:56:30.206Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:56:30.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:30 smithi100 bash[72716]: ts=2024-07-28T19:56:30.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:56:30.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:30 smithi100 bash[72716]: ts=2024-07-28T19:56:30.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:56:32.501 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.2.service' 2024-07-28T19:56:32.527 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:32.528 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-07-28T19:56:32.528 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-07-28T19:56:32.528 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.3 2024-07-28T19:56:32.996 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:32 smithi100 systemd[1]: Stopping Ceph osd.3 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:32.997 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:32 smithi100 bash[104222]: debug 2024-07-28T19:56:32.714+0000 7fd4f4465640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:56:32.997 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:32 smithi100 bash[104222]: debug 2024-07-28T19:56:32.714+0000 7fd4f4465640 -1 osd.3 961 *** Got signal Terminated *** 2024-07-28T19:56:32.997 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:32 smithi100 bash[104222]: debug 2024-07-28T19:56:32.714+0000 7fd4f4465640 -1 osd.3 961 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:56:34.425 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:34 smithi138 bash[38605]: debug 2024-07-28T19:56:34.088+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:34.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:34 smithi138 bash[44150]: debug 2024-07-28T19:56:34.420+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:34.996 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:34 smithi100 bash[104222]: debug 2024-07-28T19:56:34.542+0000 7fd4f0a79640 -1 osd.3 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:08.819001+0000 front 2024-07-28T19:56:08.818704+0000 (oldest deadline 2024-07-28T19:56:34.118934+0000) 2024-07-28T19:56:35.396 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:35 smithi138 bash[38605]: debug 2024-07-28T19:56:35.116+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:35.806 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:35 smithi138 bash[73689]: debug 2024-07-28T19:56:35.392+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:35.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:35 smithi138 bash[33034]: debug 2024-07-28T19:56:35.500+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:35.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:35 smithi138 bash[44150]: debug 2024-07-28T19:56:35.428+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:35.996 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:35 smithi100 bash[104222]: debug 2024-07-28T19:56:35.498+0000 7fd4f0a79640 -1 osd.3 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:08.819001+0000 front 2024-07-28T19:56:08.818704+0000 (oldest deadline 2024-07-28T19:56:34.118934+0000) 2024-07-28T19:56:36.352 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:36 smithi138 bash[38605]: debug 2024-07-28T19:56:36.068+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:36.746 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:36 smithi100 bash[104222]: debug 2024-07-28T19:56:36.446+0000 7fd4f0a79640 -1 osd.3 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:08.819001+0000 front 2024-07-28T19:56:08.818704+0000 (oldest deadline 2024-07-28T19:56:34.118934+0000) 2024-07-28T19:56:36.806 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:36 smithi138 bash[33034]: debug 2024-07-28T19:56:36.496+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:36.807 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:36 smithi138 bash[73689]: debug 2024-07-28T19:56:36.352+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:36.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:36 smithi138 bash[44150]: debug 2024-07-28T19:56:36.420+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:37.425 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:37 smithi138 bash[73689]: debug 2024-07-28T19:56:37.328+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:37.426 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:37 smithi138 bash[38605]: debug 2024-07-28T19:56:37.112+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:37.746 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:37 smithi100 bash[104222]: debug 2024-07-28T19:56:37.462+0000 7fd4f0a79640 -1 osd.3 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:08.819001+0000 front 2024-07-28T19:56:08.818704+0000 (oldest deadline 2024-07-28T19:56:34.118934+0000) 2024-07-28T19:56:37.806 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:37 smithi138 bash[33034]: debug 2024-07-28T19:56:37.548+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:37.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:37 smithi138 bash[44150]: debug 2024-07-28T19:56:37.424+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:38.345 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:38 smithi138 bash[38605]: debug 2024-07-28T19:56:38.080+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:38.806 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:38 smithi138 bash[33034]: debug 2024-07-28T19:56:38.540+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:38.807 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:38 smithi138 bash[73689]: debug 2024-07-28T19:56:38.344+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:38.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:38 smithi138 bash[44150]: debug 2024-07-28T19:56:38.380+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:39.246 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:38 smithi100 bash[124454]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-3 2024-07-28T19:56:39.246 INFO:journalctl@ceph.osd.3.smithi100.stdout:Jul 28 19:56:38 smithi100 bash[124518]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-3 2024-07-28T19:56:39.340 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:39 smithi138 bash[38605]: debug 2024-07-28T19:56:39.084+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:39.806 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:39 smithi138 bash[73689]: debug 2024-07-28T19:56:39.372+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:39.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:39 smithi138 bash[33034]: debug 2024-07-28T19:56:39.548+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:39.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:39 smithi138 bash[44150]: debug 2024-07-28T19:56:39.340+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:40.496 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:40 smithi100 bash[72716]: ts=2024-07-28T19:56:40.207Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:56:40.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:40 smithi100 bash[72716]: ts=2024-07-28T19:56:40.208Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:56:40.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:40 smithi100 bash[72716]: ts=2024-07-28T19:56:40.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:56:40.556 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:40 smithi138 bash[73689]: debug 2024-07-28T19:56:40.332+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:40.557 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:40 smithi138 bash[38605]: debug 2024-07-28T19:56:40.120+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:40.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:40 smithi138 bash[44150]: debug 2024-07-28T19:56:40.328+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:40.859 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.3.service' 2024-07-28T19:56:40.886 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:40.886 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-07-28T19:56:40.886 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-07-28T19:56:40.886 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.4 2024-07-28T19:56:40.898 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:40 smithi138 bash[33034]: debug 2024-07-28T19:56:40.572+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:41.160 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:40 smithi138 systemd[1]: Stopping Ceph osd.4 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:41.160 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:41 smithi138 bash[73689]: debug 2024-07-28T19:56:41.076+0000 7f5fd2f40640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:56:41.160 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:41 smithi138 bash[73689]: debug 2024-07-28T19:56:41.076+0000 7f5fd2f40640 -1 osd.4 961 *** Got signal Terminated *** 2024-07-28T19:56:41.160 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:41 smithi138 bash[73689]: debug 2024-07-28T19:56:41.076+0000 7f5fd2f40640 -1 osd.4 961 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:56:41.556 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:41 smithi138 bash[38605]: debug 2024-07-28T19:56:41.156+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:41.557 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:41 smithi138 bash[73689]: debug 2024-07-28T19:56:41.320+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:41.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:41 smithi138 bash[44150]: debug 2024-07-28T19:56:41.308+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:42.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:41 smithi138 bash[33034]: debug 2024-07-28T19:56:41.608+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:42.556 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:42 smithi138 bash[38605]: debug 2024-07-28T19:56:42.172+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:42.557 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:42 smithi138 bash[73689]: debug 2024-07-28T19:56:42.356+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:42.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:42 smithi138 bash[44150]: debug 2024-07-28T19:56:42.292+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:43.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:42 smithi138 bash[33034]: debug 2024-07-28T19:56:42.652+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:43.556 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:43 smithi138 bash[38605]: debug 2024-07-28T19:56:43.160+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:43.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:43 smithi138 bash[44150]: debug 2024-07-28T19:56:43.300+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:43.557 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:43 smithi138 bash[73689]: debug 2024-07-28T19:56:43.404+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:43.557 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:43 smithi138 bash[73689]: debug 2024-07-28T19:56:43.404+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:15.389481+0000 front 2024-07-28T19:56:15.389512+0000 (oldest deadline 2024-07-28T19:56:42.990338+0000) 2024-07-28T19:56:44.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:43 smithi138 bash[33034]: debug 2024-07-28T19:56:43.612+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:44.556 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:44 smithi138 bash[38605]: debug 2024-07-28T19:56:44.208+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:44.557 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:44 smithi138 bash[38605]: debug 2024-07-28T19:56:44.208+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:44.557 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:44 smithi138 bash[73689]: debug 2024-07-28T19:56:44.392+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:44.557 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:44 smithi138 bash[73689]: debug 2024-07-28T19:56:44.392+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:15.389481+0000 front 2024-07-28T19:56:15.389512+0000 (oldest deadline 2024-07-28T19:56:42.990338+0000) 2024-07-28T19:56:44.558 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:44 smithi138 bash[44150]: debug 2024-07-28T19:56:44.336+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:44.558 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:44 smithi138 bash[44150]: debug 2024-07-28T19:56:44.336+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:45.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:44 smithi138 bash[33034]: debug 2024-07-28T19:56:44.612+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:45.556 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:45 smithi138 bash[38605]: debug 2024-07-28T19:56:45.156+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:45.557 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:45 smithi138 bash[38605]: debug 2024-07-28T19:56:45.156+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:45.557 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:45 smithi138 bash[73689]: debug 2024-07-28T19:56:45.348+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.788167+0000 front 2024-07-28T19:56:10.788689+0000 (oldest deadline 2024-07-28T19:56:34.888267+0000) 2024-07-28T19:56:45.557 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:45 smithi138 bash[73689]: debug 2024-07-28T19:56:45.348+0000 7f5fcf554640 -1 osd.4 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:15.389481+0000 front 2024-07-28T19:56:15.389512+0000 (oldest deadline 2024-07-28T19:56:42.990338+0000) 2024-07-28T19:56:45.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:45 smithi138 bash[44150]: debug 2024-07-28T19:56:45.304+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:45.558 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:45 smithi138 bash[44150]: debug 2024-07-28T19:56:45.304+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:46.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:45 smithi138 bash[33034]: debug 2024-07-28T19:56:45.600+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:46.057 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:45 smithi138 bash[33034]: debug 2024-07-28T19:56:45.600+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:46.489 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:46 smithi138 bash[44150]: debug 2024-07-28T19:56:46.312+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:46.489 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:46 smithi138 bash[44150]: debug 2024-07-28T19:56:46.312+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:46.489 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:46 smithi138 bash[38605]: debug 2024-07-28T19:56:46.168+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:46.489 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:46 smithi138 bash[38605]: debug 2024-07-28T19:56:46.168+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:46.806 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:46 smithi138 bash[33034]: debug 2024-07-28T19:56:46.624+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:46.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:46 smithi138 bash[33034]: debug 2024-07-28T19:56:46.624+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:47.147 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:47 smithi138 bash[81965]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-4 2024-07-28T19:56:47.147 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:47 smithi138 bash[38605]: debug 2024-07-28T19:56:47.144+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:47.306 INFO:journalctl@ceph.osd.4.smithi138.stdout:Jul 28 19:56:47 smithi138 bash[82030]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-4 2024-07-28T19:56:47.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:47 smithi138 bash[38605]: debug 2024-07-28T19:56:47.144+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:47.585 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:47 smithi138 bash[44150]: debug 2024-07-28T19:56:47.328+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:47.586 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:47 smithi138 bash[44150]: debug 2024-07-28T19:56:47.328+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:48.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:47 smithi138 bash[33034]: debug 2024-07-28T19:56:47.584+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:48.057 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:47 smithi138 bash[33034]: debug 2024-07-28T19:56:47.584+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:48.414 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:48 smithi138 bash[38605]: debug 2024-07-28T19:56:48.128+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:48.414 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:48 smithi138 bash[38605]: debug 2024-07-28T19:56:48.128+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:48.414 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:48 smithi138 bash[44150]: debug 2024-07-28T19:56:48.360+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:48.415 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:48 smithi138 bash[44150]: debug 2024-07-28T19:56:48.360+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:48.737 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:48 smithi138 bash[33034]: debug 2024-07-28T19:56:48.588+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:48.738 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:48 smithi138 bash[33034]: debug 2024-07-28T19:56:48.588+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:49.306 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[38605]: debug 2024-07-28T19:56:49.108+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:49.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[38605]: debug 2024-07-28T19:56:49.108+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:49.494 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.4.service' 2024-07-28T19:56:49.563 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:49.563 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-07-28T19:56:49.563 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-07-28T19:56:49.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.5 2024-07-28T19:56:49.751 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[44150]: debug 2024-07-28T19:56:49.396+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:49.751 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[44150]: debug 2024-07-28T19:56:49.396+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:49.752 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:49 smithi138 systemd[1]: Stopping Ceph osd.5 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:49.752 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[33034]: debug 2024-07-28T19:56:49.632+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:49.752 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[33034]: debug 2024-07-28T19:56:49.632+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:49.752 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[33034]: debug 2024-07-28T19:56:49.748+0000 7fa26543b640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:56:49.752 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[33034]: debug 2024-07-28T19:56:49.748+0000 7fa26543b640 -1 osd.5 961 *** Got signal Terminated *** 2024-07-28T19:56:49.752 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:49 smithi138 bash[33034]: debug 2024-07-28T19:56:49.748+0000 7fa26543b640 -1 osd.5 961 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:56:50.448 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:50 smithi138 bash[38605]: debug 2024-07-28T19:56:50.124+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:50.448 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:50 smithi138 bash[38605]: debug 2024-07-28T19:56:50.124+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:50.496 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:50 smithi100 bash[72716]: ts=2024-07-28T19:56:50.208Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:56:50.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:50 smithi100 bash[72716]: ts=2024-07-28T19:56:50.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:56:50.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:56:50 smithi100 bash[72716]: ts=2024-07-28T19:56:50.210Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:56:50.806 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:50 smithi138 bash[33034]: debug 2024-07-28T19:56:50.624+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:50.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:50 smithi138 bash[33034]: debug 2024-07-28T19:56:50.624+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:50.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:50 smithi138 bash[44150]: debug 2024-07-28T19:56:50.448+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:50.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:50 smithi138 bash[44150]: debug 2024-07-28T19:56:50.448+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:50.808 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:50 smithi138 bash[44150]: debug 2024-07-28T19:56:50.448+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:51.441 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:51 smithi138 bash[38605]: debug 2024-07-28T19:56:51.140+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:51.441 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:51 smithi138 bash[38605]: debug 2024-07-28T19:56:51.140+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:51.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:51 smithi138 bash[44150]: debug 2024-07-28T19:56:51.436+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:51.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:51 smithi138 bash[44150]: debug 2024-07-28T19:56:51.436+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:51.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:51 smithi138 bash[44150]: debug 2024-07-28T19:56:51.436+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:51.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:51 smithi138 bash[33034]: debug 2024-07-28T19:56:51.620+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:51.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:51 smithi138 bash[33034]: debug 2024-07-28T19:56:51.620+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:51.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:51 smithi138 bash[33034]: debug 2024-07-28T19:56:51.620+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:26.965334+0000 front 2024-07-28T19:56:26.965490+0000 (oldest deadline 2024-07-28T19:56:51.065219+0000) 2024-07-28T19:56:52.409 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:52 smithi138 bash[38605]: debug 2024-07-28T19:56:52.152+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:52.409 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:52 smithi138 bash[38605]: debug 2024-07-28T19:56:52.152+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:52.806 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:52 smithi138 bash[33034]: debug 2024-07-28T19:56:52.608+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:52.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:52 smithi138 bash[33034]: debug 2024-07-28T19:56:52.608+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:52.807 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:52 smithi138 bash[33034]: debug 2024-07-28T19:56:52.608+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:26.965334+0000 front 2024-07-28T19:56:26.965490+0000 (oldest deadline 2024-07-28T19:56:51.065219+0000) 2024-07-28T19:56:52.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:52 smithi138 bash[44150]: debug 2024-07-28T19:56:52.404+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:52.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:52 smithi138 bash[44150]: debug 2024-07-28T19:56:52.404+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:52.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:52 smithi138 bash[44150]: debug 2024-07-28T19:56:52.404+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:53.366 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:53 smithi138 bash[38605]: debug 2024-07-28T19:56:53.111+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:53.366 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:53 smithi138 bash[38605]: debug 2024-07-28T19:56:53.111+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:53.617 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:53 smithi138 bash[44150]: debug 2024-07-28T19:56:53.367+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:53.617 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:53 smithi138 bash[44150]: debug 2024-07-28T19:56:53.367+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:53.617 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:53 smithi138 bash[44150]: debug 2024-07-28T19:56:53.367+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:54.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:53 smithi138 bash[33034]: debug 2024-07-28T19:56:53.615+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:54.057 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:53 smithi138 bash[33034]: debug 2024-07-28T19:56:53.615+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:54.057 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:53 smithi138 bash[33034]: debug 2024-07-28T19:56:53.615+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:26.965334+0000 front 2024-07-28T19:56:26.965490+0000 (oldest deadline 2024-07-28T19:56:51.065219+0000) 2024-07-28T19:56:54.347 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:54 smithi138 bash[38605]: debug 2024-07-28T19:56:54.059+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:54.348 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:54 smithi138 bash[38605]: debug 2024-07-28T19:56:54.059+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:54.645 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:54 smithi138 bash[44150]: debug 2024-07-28T19:56:54.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:54.645 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:54 smithi138 bash[44150]: debug 2024-07-28T19:56:54.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:54.645 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:54 smithi138 bash[44150]: debug 2024-07-28T19:56:54.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:55.056 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:54 smithi138 bash[33034]: debug 2024-07-28T19:56:54.639+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:10.562773+0000 front 2024-07-28T19:56:10.562704+0000 (oldest deadline 2024-07-28T19:56:34.662656+0000) 2024-07-28T19:56:55.057 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:54 smithi138 bash[33034]: debug 2024-07-28T19:56:54.639+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:20.564352+0000 front 2024-07-28T19:56:20.564508+0000 (oldest deadline 2024-07-28T19:56:44.664027+0000) 2024-07-28T19:56:55.057 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:54 smithi138 bash[33034]: debug 2024-07-28T19:56:54.639+0000 7fa25ea4d640 -1 osd.5 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:26.965334+0000 front 2024-07-28T19:56:26.965490+0000 (oldest deadline 2024-07-28T19:56:51.065219+0000) 2024-07-28T19:56:55.377 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:55 smithi138 bash[38605]: debug 2024-07-28T19:56:55.067+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:55.377 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:55 smithi138 bash[38605]: debug 2024-07-28T19:56:55.067+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:55.377 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:55 smithi138 bash[38605]: debug 2024-07-28T19:56:55.067+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:56:55.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:55 smithi138 bash[44150]: debug 2024-07-28T19:56:55.371+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:55.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:55 smithi138 bash[44150]: debug 2024-07-28T19:56:55.371+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:55.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:55 smithi138 bash[44150]: debug 2024-07-28T19:56:55.371+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:56.306 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:55 smithi138 bash[82158]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-5 2024-07-28T19:56:56.307 INFO:journalctl@ceph.osd.5.smithi138.stdout:Jul 28 19:56:56 smithi138 bash[82223]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-5 2024-07-28T19:56:56.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:56 smithi138 bash[38605]: debug 2024-07-28T19:56:56.043+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:56.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:56 smithi138 bash[38605]: debug 2024-07-28T19:56:56.043+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:56.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:56 smithi138 bash[38605]: debug 2024-07-28T19:56:56.043+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:56:56.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:56 smithi138 bash[44150]: debug 2024-07-28T19:56:56.339+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:56.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:56 smithi138 bash[44150]: debug 2024-07-28T19:56:56.339+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:56.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:56 smithi138 bash[44150]: debug 2024-07-28T19:56:56.339+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:57.306 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:57 smithi138 bash[38605]: debug 2024-07-28T19:56:57.003+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:57.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:57 smithi138 bash[38605]: debug 2024-07-28T19:56:57.003+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:57.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:57 smithi138 bash[38605]: debug 2024-07-28T19:56:57.003+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:56:57.675 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:57 smithi138 bash[44150]: debug 2024-07-28T19:56:57.303+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:57.675 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:57 smithi138 bash[44150]: debug 2024-07-28T19:56:57.303+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:57.675 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:57 smithi138 bash[44150]: debug 2024-07-28T19:56:57.303+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:58.138 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.5.service' 2024-07-28T19:56:58.164 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:56:58.165 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-07-28T19:56:58.165 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-07-28T19:56:58.165 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.6 2024-07-28T19:56:58.288 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[38605]: debug 2024-07-28T19:56:58.015+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:58.289 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[38605]: debug 2024-07-28T19:56:58.015+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:58.289 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[38605]: debug 2024-07-28T19:56:58.015+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:56:58.289 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:58 smithi138 systemd[1]: Stopping Ceph osd.6 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:56:58.556 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[38605]: debug 2024-07-28T19:56:58.359+0000 7f117346a640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:56:58.557 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[38605]: debug 2024-07-28T19:56:58.359+0000 7f117346a640 -1 osd.6 961 *** Got signal Terminated *** 2024-07-28T19:56:58.557 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[38605]: debug 2024-07-28T19:56:58.359+0000 7f117346a640 -1 osd.6 961 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:56:58.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[44150]: debug 2024-07-28T19:56:58.283+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:58.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[44150]: debug 2024-07-28T19:56:58.283+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:58.557 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:58 smithi138 bash[44150]: debug 2024-07-28T19:56:58.283+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:56:59.338 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:59 smithi138 bash[38605]: debug 2024-07-28T19:56:59.055+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:56:59.338 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:59 smithi138 bash[38605]: debug 2024-07-28T19:56:59.055+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:56:59.338 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:56:59 smithi138 bash[38605]: debug 2024-07-28T19:56:59.055+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:56:59.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:59 smithi138 bash[44150]: debug 2024-07-28T19:56:59.339+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:56:59.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:59 smithi138 bash[44150]: debug 2024-07-28T19:56:59.339+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:56:59.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:56:59 smithi138 bash[44150]: debug 2024-07-28T19:56:59.339+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:00.306 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[38605]: debug 2024-07-28T19:57:00.027+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:57:00.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[38605]: debug 2024-07-28T19:57:00.027+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:57:00.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[38605]: debug 2024-07-28T19:57:00.027+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:57:00.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[38605]: debug 2024-07-28T19:57:00.027+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:34.797665+0000 front 2024-07-28T19:56:34.797548+0000 (oldest deadline 2024-07-28T19:56:59.497393+0000) 2024-07-28T19:57:00.496 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:00 smithi100 bash[72716]: ts=2024-07-28T19:57:00.209Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:57:00.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:00 smithi100 bash[72716]: ts=2024-07-28T19:57:00.210Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:57:00.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:00 smithi100 bash[72716]: ts=2024-07-28T19:57:00.212Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:57:00.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[44150]: debug 2024-07-28T19:57:00.371+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:00.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[44150]: debug 2024-07-28T19:57:00.371+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:00.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[44150]: debug 2024-07-28T19:57:00.371+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:00.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[60615]: ts=2024-07-28T19:57:00.347Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.100:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.100:8765: connect: connection refused" 2024-07-28T19:57:00.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[60615]: ts=2024-07-28T19:57:00.352Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.100:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.100:8765: connect: connection refused" 2024-07-28T19:57:00.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[60615]: ts=2024-07-28T19:57:00.352Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.100:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.100:8765: connect: connection refused" 2024-07-28T19:57:00.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[60615]: ts=2024-07-28T19:57:00.352Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.100:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.100:8765: connect: connection refused" 2024-07-28T19:57:00.807 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:00 smithi138 bash[60615]: ts=2024-07-28T19:57:00.354Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.100:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.100:8765: connect: connection refused" 2024-07-28T19:57:01.306 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:01 smithi138 bash[38605]: debug 2024-07-28T19:57:01.039+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:57:01.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:01 smithi138 bash[38605]: debug 2024-07-28T19:57:01.039+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:57:01.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:01 smithi138 bash[38605]: debug 2024-07-28T19:57:01.039+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:57:01.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:01 smithi138 bash[38605]: debug 2024-07-28T19:57:01.039+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:34.797665+0000 front 2024-07-28T19:56:34.797548+0000 (oldest deadline 2024-07-28T19:56:59.497393+0000) 2024-07-28T19:57:01.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:01 smithi138 bash[44150]: debug 2024-07-28T19:57:01.391+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:01.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:01 smithi138 bash[44150]: debug 2024-07-28T19:57:01.391+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:01.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:01 smithi138 bash[44150]: debug 2024-07-28T19:57:01.391+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:02.373 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:02 smithi138 bash[38605]: debug 2024-07-28T19:57:02.071+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:57:02.374 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:02 smithi138 bash[38605]: debug 2024-07-28T19:57:02.071+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:57:02.374 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:02 smithi138 bash[38605]: debug 2024-07-28T19:57:02.071+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:57:02.374 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:02 smithi138 bash[38605]: debug 2024-07-28T19:57:02.071+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:34.797665+0000 front 2024-07-28T19:56:34.797548+0000 (oldest deadline 2024-07-28T19:56:59.497393+0000) 2024-07-28T19:57:02.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:02 smithi138 bash[44150]: debug 2024-07-28T19:57:02.375+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:02.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:02 smithi138 bash[44150]: debug 2024-07-28T19:57:02.375+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:02.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:02 smithi138 bash[44150]: debug 2024-07-28T19:57:02.375+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:03.306 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:03 smithi138 bash[38605]: debug 2024-07-28T19:57:03.035+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.693117+0000 front 2024-07-28T19:56:09.693028+0000 (oldest deadline 2024-07-28T19:56:33.192875+0000) 2024-07-28T19:57:03.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:03 smithi138 bash[38605]: debug 2024-07-28T19:57:03.035+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:19.094235+0000 front 2024-07-28T19:56:19.094425+0000 (oldest deadline 2024-07-28T19:56:43.694887+0000) 2024-07-28T19:57:03.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:03 smithi138 bash[38605]: debug 2024-07-28T19:57:03.035+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:28.996300+0000 front 2024-07-28T19:56:28.996373+0000 (oldest deadline 2024-07-28T19:56:54.296027+0000) 2024-07-28T19:57:03.307 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:03 smithi138 bash[38605]: debug 2024-07-28T19:57:03.035+0000 7f116fa82640 -1 osd.6 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:34.797665+0000 front 2024-07-28T19:56:34.797548+0000 (oldest deadline 2024-07-28T19:56:59.497393+0000) 2024-07-28T19:57:03.799 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:03 smithi138 bash[44150]: debug 2024-07-28T19:57:03.407+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:03.799 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:03 smithi138 bash[44150]: debug 2024-07-28T19:57:03.407+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:03.799 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:03 smithi138 bash[44150]: debug 2024-07-28T19:57:03.407+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:03.799 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:03 smithi138 bash[44150]: debug 2024-07-28T19:57:03.407+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:04.620 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:04 smithi138 bash[44150]: debug 2024-07-28T19:57:04.415+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:04.620 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:04 smithi138 bash[44150]: debug 2024-07-28T19:57:04.415+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:04.620 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:04 smithi138 bash[44150]: debug 2024-07-28T19:57:04.415+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:04.620 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:04 smithi138 bash[44150]: debug 2024-07-28T19:57:04.415+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:04.620 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:04 smithi138 bash[82352]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-6 2024-07-28T19:57:04.621 INFO:journalctl@ceph.osd.6.smithi138.stdout:Jul 28 19:57:04 smithi138 bash[82420]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-6 2024-07-28T19:57:05.784 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:05 smithi138 bash[44150]: debug 2024-07-28T19:57:05.367+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:05.784 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:05 smithi138 bash[44150]: debug 2024-07-28T19:57:05.367+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:05.785 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:05 smithi138 bash[44150]: debug 2024-07-28T19:57:05.367+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:05.785 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:05 smithi138 bash[44150]: debug 2024-07-28T19:57:05.367+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:06.449 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.6.service' 2024-07-28T19:57:06.477 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:57:06.478 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-07-28T19:57:06.478 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-07-28T19:57:06.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.7 2024-07-28T19:57:06.644 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:06 smithi138 bash[44150]: debug 2024-07-28T19:57:06.355+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:06.644 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:06 smithi138 bash[44150]: debug 2024-07-28T19:57:06.355+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:06.644 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:06 smithi138 bash[44150]: debug 2024-07-28T19:57:06.355+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:06.644 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:06 smithi138 bash[44150]: debug 2024-07-28T19:57:06.355+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:06.644 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:06 smithi138 systemd[1]: Stopping Ceph osd.7 for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:57:06.645 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:06 smithi138 bash[44150]: debug 2024-07-28T19:57:06.643+0000 7fc3886b3640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-07-28T19:57:06.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:06 smithi138 bash[44150]: debug 2024-07-28T19:57:06.643+0000 7fc3886b3640 -1 osd.7 961 *** Got signal Terminated *** 2024-07-28T19:57:06.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:06 smithi138 bash[44150]: debug 2024-07-28T19:57:06.643+0000 7fc3886b3640 -1 osd.7 961 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-07-28T19:57:07.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:07 smithi138 bash[44150]: debug 2024-07-28T19:57:07.311+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:07.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:07 smithi138 bash[44150]: debug 2024-07-28T19:57:07.311+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:07.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:07 smithi138 bash[44150]: debug 2024-07-28T19:57:07.311+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:07.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:07 smithi138 bash[44150]: debug 2024-07-28T19:57:07.311+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:08.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:08 smithi138 bash[44150]: debug 2024-07-28T19:57:08.351+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:08.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:08 smithi138 bash[44150]: debug 2024-07-28T19:57:08.351+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:08.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:08 smithi138 bash[44150]: debug 2024-07-28T19:57:08.351+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:08.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:08 smithi138 bash[44150]: debug 2024-07-28T19:57:08.351+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:09.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:09 smithi138 bash[44150]: debug 2024-07-28T19:57:09.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:09.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:09 smithi138 bash[44150]: debug 2024-07-28T19:57:09.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:09.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:09 smithi138 bash[44150]: debug 2024-07-28T19:57:09.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:09.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:09 smithi138 bash[44150]: debug 2024-07-28T19:57:09.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:10.496 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:10 smithi100 bash[72716]: ts=2024-07-28T19:57:10.210Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:57:10.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:10 smithi100 bash[72716]: ts=2024-07-28T19:57:10.211Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:57:10.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:10 smithi100 bash[72716]: ts=2024-07-28T19:57:10.212Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:57:10.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:10 smithi138 bash[44150]: debug 2024-07-28T19:57:10.351+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:10.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:10 smithi138 bash[44150]: debug 2024-07-28T19:57:10.351+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:10.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:10 smithi138 bash[44150]: debug 2024-07-28T19:57:10.351+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:10.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:10 smithi138 bash[44150]: debug 2024-07-28T19:57:10.351+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:11.806 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:11 smithi138 bash[44150]: debug 2024-07-28T19:57:11.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6806 osd.0 since back 2024-07-28T19:56:09.856933+0000 front 2024-07-28T19:56:09.856956+0000 (oldest deadline 2024-07-28T19:56:33.956428+0000) 2024-07-28T19:57:11.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:11 smithi138 bash[44150]: debug 2024-07-28T19:57:11.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6814 osd.1 since back 2024-07-28T19:56:17.458138+0000 front 2024-07-28T19:56:17.457893+0000 (oldest deadline 2024-07-28T19:56:43.357727+0000) 2024-07-28T19:57:11.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:11 smithi138 bash[44150]: debug 2024-07-28T19:57:11.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6822 osd.2 since back 2024-07-28T19:56:27.959898+0000 front 2024-07-28T19:56:27.959783+0000 (oldest deadline 2024-07-28T19:56:49.659484+0000) 2024-07-28T19:57:11.807 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:11 smithi138 bash[44150]: debug 2024-07-28T19:57:11.347+0000 7fc3844ca640 -1 osd.7 961 heartbeat_check: no reply from 172.21.15.100:6830 osd.3 since back 2024-07-28T19:56:35.560845+0000 front 2024-07-28T19:56:35.560882+0000 (oldest deadline 2024-07-28T19:57:03.160959+0000) 2024-07-28T19:57:13.082 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:12 smithi138 bash[82549]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-7 2024-07-28T19:57:13.082 INFO:journalctl@ceph.osd.7.smithi138.stdout:Jul 28 19:57:12 smithi138 bash[82613]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-osd-7 2024-07-28T19:57:14.784 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@osd.7.service' 2024-07-28T19:57:14.811 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:57:14.811 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-07-28T19:57:14.811 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-07-28T19:57:14.811 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-9170a172-4d15-11ef-bcc8-c7b262605968@prometheus.a 2024-07-28T19:57:15.042 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:14 smithi138 systemd[1]: Stopping Ceph prometheus.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:57:15.043 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.001Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-07-28T19:57:15.043 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.001Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-07-28T19:57:15.043 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.001Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-07-28T19:57:15.043 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.001Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-07-28T19:57:15.043 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.001Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-07-28T19:57:15.043 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.001Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-07-28T19:57:15.043 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.001Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-07-28T19:57:15.043 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.001Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-07-28T19:57:15.044 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.002Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-07-28T19:57:15.044 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.002Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-07-28T19:57:15.044 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.002Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-07-28T19:57:15.044 INFO:journalctl@ceph.prometheus.a.smithi138.stdout:Jul 28 19:57:15 smithi138 bash[60615]: ts=2024-07-28T19:57:15.002Z caller=main.go:1147 level=info msg="See you next time!" 2024-07-28T19:57:15.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9170a172-4d15-11ef-bcc8-c7b262605968@prometheus.a.service' 2024-07-28T19:57:15.747 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-07-28T19:57:15.747 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-07-28T19:57:15.747 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 --force --keep-logs 2024-07-28T19:57:20.496 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:20 smithi100 bash[72716]: ts=2024-07-28T19:57:20.211Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:57:20.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:20 smithi100 bash[72716]: ts=2024-07-28T19:57:20.212Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi100.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.100:8443: connect: connection refused" 2024-07-28T19:57:20.497 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:20 smithi100 bash[72716]: ts=2024-07-28T19:57:20.213Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi138.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.138:8443: connect: connection refused" 2024-07-28T19:57:20.769 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:20 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:20.769 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:20 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.098 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:20 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.099 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:20 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.376 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.376 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.377 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.377 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.636 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.637 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.970 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:21.971 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:22.246 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:22.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:21 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:22.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:22 smithi100 systemd[1]: Stopping Ceph node-exporter.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:57:22.977 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:22 smithi100 bash[124878]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-node-exporter-a 2024-07-28T19:57:22.977 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:22 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:57:22.977 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:22 smithi100 bash[124940]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-node-exporter-a 2024-07-28T19:57:22.977 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:22 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-07-28T19:57:22.977 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:22 smithi100 systemd[1]: Stopped Ceph node-exporter.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:57:23.246 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:22 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:23.247 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:23 smithi100 systemd[1]: Stopping Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:57:23.247 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:23 smithi100 bash[72716]: ts=2024-07-28T19:57:23.194Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-07-28T19:57:23.247 INFO:journalctl@ceph.node-exporter.a.smithi100.stdout:Jul 28 19:57:23 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:24.135 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:23 smithi100 bash[125004]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-alertmanager-a 2024-07-28T19:57:24.135 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:23 smithi100 bash[125065]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-alertmanager-a 2024-07-28T19:57:24.135 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:24 smithi100 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-07-28T19:57:24.135 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:24 smithi100 systemd[1]: Stopped Ceph alertmanager.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:57:24.496 INFO:journalctl@ceph.alertmanager.a.smithi100.stdout:Jul 28 19:57:24 smithi100 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:28.020 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 --force --keep-logs 2024-07-28T19:57:33.046 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:32 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.046 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:32 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.306 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.306 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.645 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.646 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.934 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.934 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.934 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:33.934 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:33 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:34.206 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:34 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:34.206 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:34 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:34.556 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:34 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:34.557 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:34 smithi138 systemd[1]: Stopping Ceph node-exporter.b for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:57:34.557 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:34 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:35.800 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:35 smithi138 bash[83066]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-node-exporter-b 2024-07-28T19:57:35.800 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:35 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:57:35.800 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:35 smithi138 bash[83127]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-node-exporter-b 2024-07-28T19:57:35.800 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:35 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-07-28T19:57:35.800 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:35 smithi138 systemd[1]: Stopped Ceph node-exporter.b for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:57:36.056 INFO:journalctl@ceph.node-exporter.b.smithi138.stdout:Jul 28 19:57:35 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:36.056 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:35 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:36.057 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:35 smithi138 systemd[1]: Stopping Ceph grafana.a for 9170a172-4d15-11ef-bcc8-c7b262605968... 2024-07-28T19:57:37.162 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:36 smithi138 bash[83190]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-grafana-a 2024-07-28T19:57:37.162 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:37 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-07-28T19:57:37.163 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:37 smithi138 bash[83253]: Error response from daemon: No such container: ceph-9170a172-4d15-11ef-bcc8-c7b262605968-grafana-a 2024-07-28T19:57:37.163 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:37 smithi138 systemd[1]: ceph-9170a172-4d15-11ef-bcc8-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-07-28T19:57:37.163 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:37 smithi138 systemd[1]: Stopped Ceph grafana.a for 9170a172-4d15-11ef-bcc8-c7b262605968. 2024-07-28T19:57:37.556 INFO:journalctl@ceph.grafana.a.smithi138.stdout:Jul 28 19:57:37 smithi138 systemd[1]: /etc/systemd/system/ceph-9170a172-4d15-11ef-bcc8-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-07-28T19:57:47.137 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-07-28T19:57:47.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-07-28T19:57:47.163 INFO:tasks.cephadm:Archiving crash dumps... 2024-07-28T19:57:47.165 DEBUG:teuthology.misc:Transferring archived files from smithi100:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560/remote/smithi100/crash 2024-07-28T19:57:47.166 DEBUG:teuthology.orchestra.run.smithi100:> sudo tar c -f - -C /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/crash -- . 2024-07-28T19:57:47.204 INFO:teuthology.orchestra.run.smithi100.stderr:tar: /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/crash: Cannot open: No such file or directory 2024-07-28T19:57:47.204 INFO:teuthology.orchestra.run.smithi100.stderr:tar: Error is not recoverable: exiting now 2024-07-28T19:57:47.206 DEBUG:teuthology.misc:Transferring archived files from smithi138:/var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560/remote/smithi138/crash 2024-07-28T19:57:47.207 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/crash -- . 2024-07-28T19:57:47.219 INFO:teuthology.orchestra.run.smithi138.stderr:tar: /var/lib/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/crash: Cannot open: No such file or directory 2024-07-28T19:57:47.219 INFO:teuthology.orchestra.run.smithi138.stderr:tar: Error is not recoverable: exiting now 2024-07-28T19:57:47.220 INFO:tasks.cephadm:Checking cluster log for badness... 2024-07-28T19:57:47.220 DEBUG:teuthology.orchestra.run.smithi100:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-07-28T19:57:47.260 INFO:teuthology.orchestra.run.smithi100.stderr:grep: /var/log/ceph/9170a172-4d15-11ef-bcc8-c7b262605968/ceph.log: No such file or directory 2024-07-28T19:57:47.262 INFO:tasks.cephadm:Compressing logs... 2024-07-28T19:57:47.262 DEBUG:teuthology.orchestra.run.smithi100:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-07-28T19:57:47.306 DEBUG:teuthology.orchestra.run.smithi138:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-07-28T19:57:47.315 INFO:teuthology.orchestra.run.smithi100.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-07-28T19:57:47.319 INFO:teuthology.orchestra.run.smithi138.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-07-28T19:58:26.205 INFO:tasks.cephadm:Archiving logs... 2024-07-28T19:58:26.206 DEBUG:teuthology.misc:Transferring archived files from smithi100:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560/remote/smithi100/log 2024-07-28T19:58:26.207 DEBUG:teuthology.orchestra.run.smithi100:> sudo tar c -f - -C /var/log/ceph -- . 2024-07-28T19:58:29.981 DEBUG:teuthology.misc:Transferring archived files from smithi138:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560/remote/smithi138/log 2024-07-28T19:58:29.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /var/log/ceph -- . 2024-07-28T19:58:33.224 INFO:tasks.cephadm:Removing cluster... 2024-07-28T19:58:33.224 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 --force 2024-07-28T19:58:34.779 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9170a172-4d15-11ef-bcc8-c7b262605968 --force 2024-07-28T19:58:36.332 INFO:tasks.cephadm:Removing cephadm ... 2024-07-28T19:58:36.332 DEBUG:teuthology.orchestra.run.smithi100:> rm -rf /home/ubuntu/cephtest/cephadm 2024-07-28T19:58:36.339 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf /home/ubuntu/cephtest/cephadm 2024-07-28T19:58:36.344 INFO:tasks.cephadm:Teardown complete 2024-07-28T19:58:36.344 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-07-28T19:58:36.354 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-07-28T19:58:36.355 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-07-28T19:58:36.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-07-28T19:58:36.442 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-07-28T19:58:36.484 INFO:teuthology.task.clock:Checking final clock skew... 2024-07-28T19:58:36.484 DEBUG:teuthology.orchestra.run.smithi100:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-07-28T19:58:36.487 DEBUG:teuthology.orchestra.run.smithi138:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-07-28T19:58:36.497 INFO:teuthology.orchestra.run.smithi100.stdout: remote refid st t when poll reach delay offset jitter 2024-07-28T19:58:36.497 INFO:teuthology.orchestra.run.smithi100.stdout:============================================================================== 2024-07-28T19:58:36.498 INFO:teuthology.orchestra.run.smithi100.stdout:*hv01.front.sepi 67.205.162.81 3 u 12 64 377 0.098 -4.478 0.931 2024-07-28T19:58:36.498 INFO:teuthology.orchestra.run.smithi100.stdout:+hv02.front.sepi 63.231.80.2 3 u 57 64 377 0.079 -0.970 1.169 2024-07-28T19:58:36.498 INFO:teuthology.orchestra.run.smithi100.stdout:+hv03.front.sepi 74.6.168.72 3 u 64 64 377 0.099 -2.305 1.232 2024-07-28T19:58:36.498 INFO:teuthology.orchestra.run.smithi100.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-07-28T19:58:36.501 INFO:teuthology.orchestra.run.smithi138.stdout: remote refid st t when poll reach delay offset jitter 2024-07-28T19:58:36.501 INFO:teuthology.orchestra.run.smithi138.stdout:============================================================================== 2024-07-28T19:58:36.501 INFO:teuthology.orchestra.run.smithi138.stdout:*hv01.front.sepi 67.205.162.81 3 u 63 64 377 0.099 -3.410 0.232 2024-07-28T19:58:36.502 INFO:teuthology.orchestra.run.smithi138.stdout:+hv02.front.sepi 63.231.80.2 3 u 16 64 377 0.092 -1.432 0.142 2024-07-28T19:58:36.502 INFO:teuthology.orchestra.run.smithi138.stdout:+hv03.front.sepi 74.6.168.72 3 u 63 64 377 0.098 -3.033 0.367 2024-07-28T19:58:36.502 INFO:teuthology.orchestra.run.smithi138.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-07-28T19:58:36.503 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-07-28T19:58:36.520 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-07-28T19:58:36.521 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-07-28T19:58:36.642 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-07-28T19:58:36.741 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-07-28T19:58:36.801 INFO:teuthology.task.internal:Duration was 3192.385484 seconds 2024-07-28T19:58:36.802 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-07-28T19:58:36.941 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-07-28T19:58:36.941 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-07-28T19:58:36.943 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-07-28T19:58:36.979 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-07-28T19:58:36.980 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi100.front.sepia.ceph.com 2024-07-28T19:58:36.980 DEBUG:teuthology.orchestra.run.smithi100:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-07-28T19:58:37.030 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi138.front.sepia.ceph.com 2024-07-28T19:58:37.030 DEBUG:teuthology.orchestra.run.smithi138:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-07-28T19:58:37.040 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-07-28T19:58:37.041 DEBUG:teuthology.orchestra.run.smithi100:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-07-28T19:58:37.078 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-07-28T19:58:37.149 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-07-28T19:58:37.149 DEBUG:teuthology.orchestra.run.smithi100:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-07-28T19:58:37.151 DEBUG:teuthology.orchestra.run.smithi138:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-07-28T19:58:37.358 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-07-28T19:58:37.368 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-07-28T19:58:37.369 DEBUG:teuthology.orchestra.run.smithi100:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-07-28T19:58:37.379 DEBUG:teuthology.orchestra.run.smithi138:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-07-28T19:58:37.391 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-07-28T19:58:37.456 DEBUG:teuthology.orchestra.run.smithi100:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-07-28T19:58:37.459 DEBUG:teuthology.orchestra.run.smithi138:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-07-28T19:58:37.467 INFO:teuthology.orchestra.run.smithi100.stdout:kernel.core_pattern = core 2024-07-28T19:58:37.471 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = core 2024-07-28T19:58:37.557 DEBUG:teuthology.orchestra.run.smithi100:> test -e /home/ubuntu/cephtest/archive/coredump 2024-07-28T19:58:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-28T19:58:37.562 DEBUG:teuthology.orchestra.run.smithi138:> test -e /home/ubuntu/cephtest/archive/coredump 2024-07-28T19:58:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-07-28T19:58:37.605 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-07-28T19:58:37.618 INFO:teuthology.task.internal:Transferring archived files... 2024-07-28T19:58:37.619 DEBUG:teuthology.misc:Transferring archived files from smithi100:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560/remote/smithi100 2024-07-28T19:58:37.619 DEBUG:teuthology.orchestra.run.smithi100:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-07-28T19:58:37.651 DEBUG:teuthology.misc:Transferring archived files from smithi138:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-07-24_01:16:02-upgrade:quincy-x-reef-distro-default-smithi/7815560/remote/smithi138 2024-07-28T19:58:37.651 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-07-28T19:58:37.677 INFO:teuthology.task.internal:Removing archive directory... 2024-07-28T19:58:37.678 DEBUG:teuthology.orchestra.run.smithi100:> rm -rf -- /home/ubuntu/cephtest/archive 2024-07-28T19:58:37.686 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf -- /home/ubuntu/cephtest/archive 2024-07-28T19:58:37.727 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-07-28T19:58:37.766 INFO:teuthology.task.internal:Not uploading archives. 2024-07-28T19:58:37.767 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-07-28T19:58:37.961 INFO:teuthology.task.internal:Tidying up after the test... 2024-07-28T19:58:37.962 DEBUG:teuthology.orchestra.run.smithi100:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-07-28T19:58:37.965 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-07-28T19:58:37.968 INFO:teuthology.orchestra.run.smithi100.stdout: 658628 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jul 28 19:58 /home/ubuntu/cephtest 2024-07-28T19:58:37.971 INFO:teuthology.orchestra.run.smithi138.stdout: 658625 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jul 28 19:58 /home/ubuntu/cephtest 2024-07-28T19:58:37.973 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-07-28T19:58:38.138 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-07-28T19:58:38.205 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 3192.3854835033417 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=50a1708004f549b4a1067057efcd10d0 status: fail success: false 2024-07-28T19:58:38.205 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-07-28T19:58:38.290 INFO:teuthology.run:FAIL